Harvest Log Status Pages

api/pagination.py

@@ -24,6 +24,7 @@ def count(self):
 class FuzzyPageNumberPagination(PageNumberPagination):
 
     django_paginator_class = FuzzyPaginator
+    max_page_size = 300
 
 
 class CursorPagination(CursorPagination):

api/serializers.py

@@ -5,10 +5,10 @@
 from rest_framework_json_api import serializers
 
 from share import models
-from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult
 
-from api import fields
+from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, logs
 
+from api import fields
 
 class BaseShareSerializer(serializers.ModelSerializer):
 
@@ -132,12 +132,20 @@ class Meta:
             'is_active', 'gravatar', 'locale', 'time_zone', 'is_trusted'
         )
 
-
 class SourceSerializer(ShareModelSerializer):
     class Meta:
         model = models.Source
         fields = ('name', 'home_page', 'long_title', 'icon')
 
+class HarvestLogSerializer(ShareModelSerializer):
+    class Meta:
+        model = models.HarvestLog
+        fields = '__all__'
+
+class SourceConfigSerializer(ShareModelSerializer):
+    class Meta:
+        model = models.SourceConfig
+        fields = '__all__'
 
 class SiteBannerSerializer(ShareModelSerializer):
     color = serializers.SerializerMethodField()

api/urls.py

@@ -14,6 +14,8 @@
 from api.serializers import BaseShareSerializer
 from api.views.share import ShareObjectViewSet
 
+
+
 app_name = 'api'
 
 router = DefaultRouter()
@@ -86,6 +88,8 @@ def register_url(self, subclass, viewset):
 register_route(r'rawdata', views.RawDatumViewSet)
 register_route(r'user', views.ShareUserViewSet)
 register_route(r'sources', views.SourceViewSet)
+register_route(r'HarvestLog', views.HarvestLogViewSet)
+register_route(r'SourceConfig', views.SourceConfigViewSet)
 
 router.register(r'normalizeddata', views.NormalizedDataViewSet, base_name='normalizeddata')
 

api/views/__init__.py

@@ -5,3 +5,5 @@
 from .registration import *  # noqa
 from .schema import *  # noqa
 from .banner import *  # noqa
+from .harvestlogs import *  # noqa
+from .sourceConfig import * # noqa

api/views/harvestlogs.py

@@ -0,0 +1,24 @@
+from rest_framework import viewsets
+from rest_framework import filters
+from django_filters.filters import MultipleChoiceFilter
+from api.views import ShareObjectViewSet
+from share.util import IDObfuscator
+
+from api.serializers import HarvestLogSerializer
+from share.models import HarvestLog
+
+class SourceConfigFilterBackend(MultipleChoiceFilter):
+    def filter_queryset(self, request, queryset, view, conjoined=True):
+        if 'source_config_id' in request.GET:
+            decoded = IDObfuscator.decode_id(request.GET['source_config_id'])
+            queryset = queryset.filter(source_config_id=decoded)
+        if 'status' in request.GET:
+            queryset = queryset.filter(status__in=request.GET.getlist('status'))
+        return queryset
+        #return queryset.order_by('endDate')
+
+class HarvestLogViewSet(ShareObjectViewSet):
+    serializer_class = HarvestLogSerializer
+    queryset = HarvestLog.objects.all()
+    filter_backends = (SourceConfigFilterBackend, )
+    filter_fields = ('status',)

api/views/share.py

@@ -28,6 +28,7 @@ class ShareObjectViewSet(viewsets.ReadOnlyModelViewSet):
 
     # Override get_queryset to handle items marked as deleted.
     def get_queryset(self, list=True):
+        # import ipdb; ipdb.set_trace()
         queryset = super().get_queryset()
         if list and hasattr(queryset.model, 'is_deleted'):
             return queryset.exclude(is_deleted=True)

api/views/sourceConfig.py

@@ -0,0 +1,33 @@
+from rest_framework import viewsets
+from api.views import ShareObjectViewSet
+from rest_framework import filters
+from api.serializers import SourceConfigSerializer
+from share.models import SourceConfig
+from api.pagination import FuzzyPageNumberPagination
+from django.db.models import Count
+from django.db.models import Aggregate
+from share.models import HarvestLog
+from django.db.models import OuterRef, Subquery
+from django.db.models import IntegerField
+
+class SourceConfigViewSet(ShareObjectViewSet):
+    serializer_class = SourceConfigSerializer
+    pagination_class = FuzzyPageNumberPagination
+    queryset = SourceConfig.objects.all()
+
+    # def get_queryset(self):
+    #     recent_harvests = HarvestLog.objects.filter(
+    #         source_config_id = OuterRef(OuterRef('id')),
+    #         status__in = [HarvestLog.STATUS.failed, HarvestLog.STATUS.succeeded]
+    #         ).order_by('-date_started')[:10]
+    #     recent_fails = HarvestLog.objects.filter(
+    #         status = HarvestLog.STATUS.failed,
+    #         id__in = Subquery(recent_harvests.values('id')),
+    #         # output_field = models.IntegerField())
+    #         )
+    #     queryset = SourceConfig.objects.all().annotate(
+    #         fails=Count(Subquery(recent_fails.values('id')),
+    #         # output_field = models.IntegerField()))
+    #         )
+    #     queryset = queryset.order_by('fails')
+    #     return queryset

bin/celery

@@ -0,0 +1,12 @@
+#!/Users/admin/SHARE/bin/python3.5
+# EASY-INSTALL-ENTRY-SCRIPT: 'celery==4.0.2','console_scripts','celery'
+__requires__ = 'celery==4.0.2'
+import re
+import sys
+from pkg_resources import load_entry_point
+
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+    sys.exit(
+        load_entry_point('celery==4.0.2', 'console_scripts', 'celery')()
+    )

docs/elasticsearch.rst

@@ -26,6 +26,7 @@ Elasticsearch can be used to search the following fields in the normalized data:
     'contributors'
     'funders'
     'publishers'
+    'id'
 
 
 Accessing the Search API

include/site/python3.5/greenlet/greenlet.h

@@ -0,0 +1,148 @@
+/* vim:set noet ts=8 sw=8 : */
+
+/* Greenlet object interface */
+
+#ifndef Py_GREENLETOBJECT_H
+#define Py_GREENLETOBJECT_H
+
+#include <Python.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GREENLET_VERSION "0.4.12"
+
+typedef struct _greenlet {
+	PyObject_HEAD
+	char* stack_start;
+	char* stack_stop;
+	char* stack_copy;
+	intptr_t stack_saved;
+	struct _greenlet* stack_prev;
+	struct _greenlet* parent;
+	PyObject* run_info;
+	struct _frame* top_frame;
+	int recursion_depth;
+	PyObject* weakreflist;
+	PyObject* exc_type;
+	PyObject* exc_value;
+	PyObject* exc_traceback;
+	PyObject* dict;
+} PyGreenlet;
+
+#define PyGreenlet_Check(op)      PyObject_TypeCheck(op, &PyGreenlet_Type)
+#define PyGreenlet_MAIN(op)       (((PyGreenlet*)(op))->stack_stop == (char*) -1)
+#define PyGreenlet_STARTED(op)    (((PyGreenlet*)(op))->stack_stop != NULL)
+#define PyGreenlet_ACTIVE(op)     (((PyGreenlet*)(op))->stack_start != NULL)
+#define PyGreenlet_GET_PARENT(op) (((PyGreenlet*)(op))->parent)
+
+#if (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION >= 7) || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 1) || PY_MAJOR_VERSION > 3
+#define GREENLET_USE_PYCAPSULE
+#endif
+
+/* C API functions */
+
+/* Total number of symbols that are exported */
+#define PyGreenlet_API_pointers  8
+
+#define PyGreenlet_Type_NUM       0
+#define PyExc_GreenletError_NUM   1
+#define PyExc_GreenletExit_NUM    2
+
+#define PyGreenlet_New_NUM        3
+#define PyGreenlet_GetCurrent_NUM 4
+#define PyGreenlet_Throw_NUM      5
+#define PyGreenlet_Switch_NUM     6
+#define PyGreenlet_SetParent_NUM  7
+
+#ifndef GREENLET_MODULE
+/* This section is used by modules that uses the greenlet C API */
+static void **_PyGreenlet_API = NULL;
+
+#define PyGreenlet_Type (*(PyTypeObject *) _PyGreenlet_API[PyGreenlet_Type_NUM])
+
+#define PyExc_GreenletError \
+	((PyObject *) _PyGreenlet_API[PyExc_GreenletError_NUM])
+
+#define PyExc_GreenletExit \
+	((PyObject *) _PyGreenlet_API[PyExc_GreenletExit_NUM])
+
+/*
+ * PyGreenlet_New(PyObject *args)
+ *
+ * greenlet.greenlet(run, parent=None)
+ */
+#define PyGreenlet_New \
+	(* (PyGreenlet * (*)(PyObject *run, PyGreenlet *parent)) \
+	 _PyGreenlet_API[PyGreenlet_New_NUM])
+
+/*
+ * PyGreenlet_GetCurrent(void)
+ *
+ * greenlet.getcurrent()
+ */
+#define PyGreenlet_GetCurrent \
+	(* (PyGreenlet * (*)(void)) _PyGreenlet_API[PyGreenlet_GetCurrent_NUM])
+
+/*
+ * PyGreenlet_Throw(
+ *         PyGreenlet *greenlet,
+ *         PyObject *typ,
+ *         PyObject *val,
+ *         PyObject *tb)
+ *
+ * g.throw(...)
+ */
+#define PyGreenlet_Throw \
+	(* (PyObject * (*) \
+	    (PyGreenlet *self, PyObject *typ, PyObject *val, PyObject *tb)) \
+	 _PyGreenlet_API[PyGreenlet_Throw_NUM])
+
+/*
+ * PyGreenlet_Switch(PyGreenlet *greenlet, PyObject *args)
+ *
+ * g.switch(*args, **kwargs)
+ */
+#define PyGreenlet_Switch \
+	(* (PyObject * (*)(PyGreenlet *greenlet, PyObject *args, PyObject *kwargs)) \
+	 _PyGreenlet_API[PyGreenlet_Switch_NUM])
+
+/*
+ * PyGreenlet_SetParent(PyObject *greenlet, PyObject *new_parent)
+ *
+ * g.parent = new_parent
+ */
+#define PyGreenlet_SetParent \
+	(* (int (*)(PyGreenlet *greenlet, PyGreenlet *nparent)) \
+	 _PyGreenlet_API[PyGreenlet_SetParent_NUM])
+
+/* Macro that imports greenlet and initializes C API */
+#ifdef GREENLET_USE_PYCAPSULE
+#define PyGreenlet_Import() \
+{ \
+	_PyGreenlet_API = (void**)PyCapsule_Import("greenlet._C_API", 0); \
+}
+#else
+#define PyGreenlet_Import() \
+{ \
+	PyObject *module = PyImport_ImportModule("greenlet"); \
+	if (module != NULL) { \
+		PyObject *c_api_object = PyObject_GetAttrString( \
+			module, "_C_API"); \
+		if (c_api_object != NULL && PyCObject_Check(c_api_object)) { \
+			_PyGreenlet_API = \
+				(void **) PyCObject_AsVoidPtr(c_api_object); \
+			Py_DECREF(c_api_object); \
+		} \
+		Py_DECREF(module); \
+	} \
+}
+#endif
+
+#endif /* GREENLET_MODULE */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_GREENLETOBJECT_H */

pip-selfcheck.json

@@ -0,0 +1 @@
+{"last_check":"2017-06-22T15:35:52Z","pypi_version":"9.0.1"}

pyvenv.cfg

@@ -0,0 +1,3 @@
+home = /usr/local/bin
+include-system-site-packages = false
+version = 3.5.3

share/doc/networkx-1.11/INSTALL.txt

@@ -0,0 +1,3 @@
+See doc/source/install.rst
+or
+http://networkx.github.io/documentation/latest/install.html

share/doc/networkx-1.11/LICENSE.txt

@@ -0,0 +1,40 @@
+License
+=======
+NetworkX is distributed with the BSD license.
+
+::
+
+   Copyright (C) 2004-2016, NetworkX Developers
+   Aric Hagberg <[email protected]>
+   Dan Schult <[email protected]>
+   Pieter Swart <[email protected]>
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+     * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+
+     * Redistributions in binary form must reproduce the above
+       copyright notice, this list of conditions and the following
+       disclaimer in the documentation and/or other materials provided
+       with the distribution.
+
+     * Neither the name of the NetworkX Developers nor the names of its
+       contributors may be used to endorse or promote products derived
+       from this software without specific prior written permission.
+
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.