Merge pull request #525 from nexB/518-update-dependencies

518 update dependencies

.dockerignore

@@ -0,0 +1,39 @@
+
+**/*.egg-info
+**/.*
+**/.*cache
+**/.DS_Store
+**/.aof
+**/.aws
+**/.git
+**/.github
+**/.gitignore
+**/.idea
+**/.ssh
+**/.venv
+**/.vscode
+**/bin
+**/build
+**/coverage
+**/dist
+**/docs
+**/env
+**/etc
+**/lib
+**/var
+**/venv
+.dockerignore
+.env
+.readthedocs.yaml
+CHANGELOG.rst
+CODE_OF_CONDUCT.rst
+Dockerfile
+MANIFEST.in
+Makefile
+README.rst
+azure-pipelines.yml
+configure
+docker*.env
+docker-compose.*yml
+pyvenv.cfg
+traefik.yml

Dockerfile

@@ -1,14 +1,23 @@
 # Copyright (c) nexB Inc. and others. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-FROM --platform=linux/amd64 python:3.11
+FROM --platform=linux/amd64 python:3.12-slim
 
-WORKDIR /app
+LABEL org.opencontainers.image.source="https://github.com/nexB/purldb"
+LABEL org.opencontainers.image.description="PurlDB"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
 
-# Python settings: Force unbuffered stdout and stderr (i.e. they are flushed to terminal immediately)
+ENV APP_NAME purldb
+ENV APP_USER app
+ENV APP_DIR /opt/$APP_NAME
+ENV VENV_LOCATION /opt/$APP_NAME/venv
+
+# Force Python unbuffered stdout and stderr (they are flushed to terminal immediately)
 ENV PYTHONUNBUFFERED 1
-# Python settings: do not write pyc files
+# Do not write Python .pyc files
 ENV PYTHONDONTWRITEBYTECODE 1
+# Add the app dir in the Python path for entry points availability
+ENV PYTHONPATH $PYTHONPATH:$APP_DIR
 
 # OS requirements as per
 # https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html
@@ -19,23 +28,54 @@ RUN apt-get update \
        zlib1g \
        libxml2-dev \
        libxslt1-dev \
+       libgomp1 \
+       libsqlite3-0 \
+       libgcrypt20 \
        libpopt0 \
-       bzip2 \
+       libzstd1 \
+       libgpgme11 \
+       libdevmapper1.02.1 \
+       libguestfs-tools \
+       linux-image-amd64 \
+       git \
+       wait-for-it \
+       universal-ctags \
+       gettext \
        tar \
        unzip \
        zip \
        libsasl2-dev \
        libldap-dev \
        openssl \
-       wait-for-it \
-       git \
        cvs \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
+# Create the APP_USER group and user
+RUN addgroup --system $APP_USER \
+ && adduser --system --group --home=$APP_DIR $APP_USER \
+ && chown $APP_USER:$APP_USER $APP_DIR
+
+# Create the /var/APP_NAME directory with proper permission for APP_USER
+RUN mkdir -p /var/$APP_NAME \
+ && chown $APP_USER:$APP_USER /var/$APP_NAME
+
+# Setup the work directory and the user as APP_USER for the remaining stages
+WORKDIR $APP_DIR
+USER $APP_USER
+
+# Create the virtualenv
+RUN python -m venv $VENV_LOCATION
+# Enable the virtualenv, similar effect as "source activate"
+ENV PATH $VENV_LOCATION/bin:$PATH
+
+# Create static/ and workspace/ directories
+RUN mkdir -p /var/$APP_NAME/static/ \
+ && mkdir -p /var/$APP_NAME/workspace/
+
 # Install the dependencies before the codebase COPY for proper Docker layer caching
-COPY setup.cfg setup.py requirements.txt /app/
-RUN pip install --no-cache-dir -c requirements.txt .
+COPY --chown=$APP_USER:$APP_USER setup.cfg setup.py $APP_DIR/
+RUN pip install --no-cache-dir .
 
-# Copy the codebase
-COPY . /app
+# Copy the codebase and set the proper permissions for the APP_USER
+COPY --chown=$APP_USER:$APP_USER . $APP_DIR

docker-compose.common.yml

@@ -5,6 +5,8 @@ services:
       - docker_purldb.env
     volumes:
       - db_data:/var/lib/postgresql/data/
+    shm_size: "1gb"
+    restart: always
 
   redis:
     image: redis
@@ -15,7 +17,7 @@ services:
 
   web:
     build: .
-    command: sh -c "
+    command: wait-for-it --strict --timeout=60 db:5432 -- sh -c "
         python manage_purldb.py migrate &&
         python manage_purldb.py collectstatic --no-input --verbosity 0 --clear &&
         gunicorn purldb_project.wsgi:application --bind :8000 --timeout 600 --workers 8"
@@ -24,9 +26,10 @@ services:
     expose:
       - 8000
     volumes:
+      - .env:/opt/purldb/.env
       - /etc/purldb/:/etc/purldb/
-      - static:/var/purldb/static/
       - temp_data:/tmp/minecode/
+      - static:/var/purldb/static/
     depends_on:
       - db
 
@@ -36,6 +39,7 @@ services:
     env_file:
       - docker_purldb.env
     volumes:
+      - .env:/opt/purldb/.env
       - /etc/purldb/:/etc/purldb/
     profiles:
       - priority_queue
@@ -49,6 +53,7 @@ services:
     env_file:
       - docker_purldb.env
     volumes:
+      - .env:/opt/purldb/.env
       - /etc/purldb/:/etc/purldb/
     depends_on:
       - redis
@@ -61,6 +66,7 @@ services:
     env_file:
       - docker_purldb.env
     volumes:
+      - .env:/opt/purldb/.env
       - /etc/purldb/:/etc/purldb/
       - temp_data:/tmp/minecode/
     depends_on:

docker-compose.yml

@@ -12,9 +12,11 @@ services:
       - "traefik.http.routers.web.entrypoints=web"
     volumes:
       - ./etc/nginx/conf.d/:/etc/nginx/conf.d/
+      - /var/www/html:/var/www/html
       - static:/var/purldb/static/
     depends_on:
       - web
+    restart: always
 
   traefik:
     image: "traefik:v2.10"

matchcode/tests/test_models.py

@@ -437,7 +437,7 @@ def test_ApproximateResourceContentIndex_match_deep_equals(self):
         matches = ApproximateResourceContentIndex.match(fp)
         results = [match.package.to_dict() for match in matches]
         expected_results_loc = self.get_test_loc('match/approximate-file-matching/index-modified.js-expected.json')
-        self.check_expected_results(results, expected_results_loc, regen=True)
+        self.check_expected_results(results, expected_results_loc, regen=FIXTURES_REGEN)
 
 
 class MatchcodeModelUtilsTestCase(MatchcodeTestCase):

minecode/mappers/apache.py

@@ -107,6 +107,7 @@ def build_packages_from_projects(metadata, uri=None):
         if license_url in APACHE_LICENSE_URL:
             common_data['declared_license_expression'] = 'apache-2.0'
             common_data['declared_license_expression_spdx'] = 'Apache-2.0'
+            common_data['license_detections'] = []
 
         keywords = []
         category = project_meta.get('category', '')

minecode/mappers/bower.py

@@ -81,6 +81,7 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
             keywords=keywords_content,
             homepage_url=content.get('homepage'),
             datasource_id='bower_json',
+            license_detections=[],
         )
 
         if extracted_license_statement:

minecode/mappers/cpan.py

@@ -57,6 +57,7 @@ def build_packages_from_release_json(metadata, uri=None):
             version=release.get('version'),
             download_url=release.get('download_url'),
             extracted_license_statement=extracted_license_statement,
+            license_detections=[],
             # the date format passing is like:
             # "2014-04-20T21:30:13"
             release_date=parse_date(release.get('date')),
@@ -177,7 +178,7 @@ def build_packages_from_metafile(metadata, uri=None, purl=None):
             # https://spdx.org/spdx-specification-21-web-version#h.49x2ik5
             vcs_repo = vcs_tool + '+' + vcs_repo
         common_data = dict(
-            datasource_id="cpan_metadata_json",
+            datasource_id="cpan_meta_json",
             type='cpan',
             name=name,
             description=content.get('abstract', name),

minecode/mappers/cran.py

@@ -20,7 +20,7 @@
 
 
 @map_router.route('https://cloud.r-project.org/web/packages/[\w\-\.]/index.html')
-class CpanMetaFileMapper(Mapper):
+class CranMetaFileMapper(Mapper):
 
     def get_packages(self, uri, resource_uri):
         """
@@ -138,6 +138,7 @@ def build_packages_from_html(metadata, uri=None, purl=None):
                     common_data['dependencies'] = get_dependencies(value)
     if extracted_license_statement:
         common_data['extracted_license_statement'] = extracted_license_statement
+        common_data['license_detections'] = []
 
     if download_urls:  # for else statement will have else running always if there is no break statement
         for download_url in download_urls:

minecode/mappers/eclipse.py

@@ -65,6 +65,7 @@ def build_packages_with_json(metadata, purl=None, uri=None):
 
         if project_metadata.get('licenses'):
             common_data['extracted_license_statement'] = [l.get('name') for l in project_metadata.get('licenses', [])]
+            common_data['license_detections'] = []
 
         # FIXME: this is a download page and NOT a download URL!!!!!
         for download_url in project_metadata.get('download_url', []):
@@ -121,6 +122,7 @@ def build_packages(html_text, purl=None, uri=None):
                 extracted_license_statement.append(license_name)
     if extracted_license_statement:
         common_data['extracted_license_statement'] = extracted_license_statement
+        common_data['license_detections'] = []
 
     for a in page.find_all(name='a'):
         if a.contents:

minecode/mappers/googlecode.py

@@ -54,6 +54,7 @@ def build_packages_from_projectsjson_v2(metadata, purl=None, uri=None):
     license_name = metadata.get('license')
     if license_name:
         common_data['extracted_license_statement'] = license_name
+        common_data['license_detections'] = []
 
     keywords = []
     labels = metadata.get('labels')
@@ -100,6 +101,7 @@ def build_packages_from_projectsjson_v1(metadata, purl=None, uri=None):
         license_name = metadata.get('license')
         if license_name:
             common_data['extracted_license_statement'] = license_name
+            common_data['license_detections'] = []
 
         keywords = []
         labels = metadata.get('labels')

minecode/mappers/nuget.py

@@ -64,6 +64,7 @@ def build_packages_with_json(metadata, purl=None):
             homepage_url=metadata.get('projectUrl'),
             description=description,
             extracted_license_statement=licenseUrl,
+            license_detections=[],
             copyright=copyr,
             parties=authors,
             keywords=keywords,
@@ -168,6 +169,7 @@ def build_packages_from_html(metadata, uri, purl=None):
                         description=description,
                         download_url=download_url,
                         extracted_license_statement=license_value,
+                        license_detections=[],
                         copyright=copyright_value
                     )
                     package = scan_models.Package.from_package_data(

minecode/mappers/openssl.py

@@ -53,6 +53,7 @@ def build_packages(resource_uri, purl=None):
         size=resource_uri.size,
         release_date=parse_date(resource_uri.last_modified_date),
         extracted_license_statement='OpenSSL License',
+        license_detections=[],
         homepage_url='https://www.openssl.org/',
         download_url=uri,
         copyright='Copyright (c) 1998-2018 The OpenSSL Project\nCopyright (c) 1995-1998 Eric A. Young, Tim J. Hudson\nAll rights reserved.',

minecode/mappers/packagist.py

@@ -42,7 +42,7 @@ def build_packages_with_json(metadata, purl=None, uri=None):
         primary_language = package.get('language')
         for version_content in package.get('versions').values():
             common = dict(
-                datasource_id='composer_json',
+                datasource_id='php_composer_json',
                 type='composer',
                 name=version_content.get('name'),
                 description=version_content.get('description'),
@@ -77,6 +77,7 @@ def build_packages_with_json(metadata, purl=None, uri=None):
                 extracted_license_statement.add(lic)
             if extracted_license_statement:
                 common['extracted_license_statement'] = list(extracted_license_statement)
+                common['license_detections'] = []
 
             dependencies = []
             for name, version in version_content.get('require', {}).items():

minecode/mappers/sourceforge.py

@@ -51,6 +51,7 @@ def build_packages_from_metafile(metadata, purl=None, uri=None):
             name=metadata.get('shortname', metadata.get('name')),
             description=description,
             homepage_url=metadata.get('external_homepage', metadata.get('url')),
+            license_detections=[],
         )
 
         devs = metadata.get('developers') or []

minecode/tests/test_cran.py

@@ -46,7 +46,7 @@ def test_build_packages_from_directory_listing(self):
         packages = mappers.cran.build_packages_from_html(metadata, 'https://cloud.r-project.org/web/packages/ANN2/index.html', 'pkg:cran/ANN2')
         packages = [p.to_dict() for p in packages]
         expected_loc = self.get_test_loc('cran/mapper_ANN2_expected.json')
-        self.check_expected_results(packages, expected_loc, regen=True)
+        self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN)
 
     def test_build_packages_from_directory_listing2(self):
         ResourceURI.objects.create(uri='https://cloud.r-project.org/web/packages/abe/index.html')
@@ -55,7 +55,7 @@ def test_build_packages_from_directory_listing2(self):
         packages = mappers.cran.build_packages_from_html(metadata, 'https://cloud.r-project.org/web/packages/abe/index.htm', 'pkg:cran/abe')
         packages = [p.to_dict() for p in packages]
         expected_loc = self.get_test_loc('cran/mapper_abe_expected.json')
-        self.check_expected_results(packages, expected_loc, regen=True)
+        self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN)
 
     def test_replace_downloadurl(self):
         url = "../../../src/contrib/Archive/ANN2"

minecode/tests/testfiles/apache/map-groovy_expected.json

@@ -31,6 +31,8 @@
   "extracted_license_statement":null,
   "notice_text":null,
   "source_packages":[],
+  "is_private":false,
+  "is_virtual":false,
   "extra_data":{},
   "repository_homepage_url":null,
   "repository_download_url":null,

minecode/tests/testfiles/apache/map-turbine-webapp_expected.json

@@ -31,6 +31,8 @@
   "extracted_license_statement":null,
   "notice_text":null,
   "source_packages":[],
+  "is_private":false,
+  "is_virtual":false,
   "extra_data":{},
   "repository_homepage_url":null,
   "repository_download_url":null,

minecode/tests/testfiles/apache/podling_amaterasu_expected.json

@@ -32,6 +32,8 @@
     "extracted_license_statement":null,
     "notice_text":null,
     "source_packages":[],
+    "is_private":false,
+    "is_virtual":false,
     "extra_data":{},
     "repository_homepage_url":null,
     "repository_download_url":null,