[rqd] [cuegui] Consolidate log read and write into a single package

ci/pylintrc_main

@@ -206,9 +206,8 @@ ignored-classes=optparse.Values,thread._local,_thread._local
 ignored-modules=opencue.compiled_proto,
             opencue.compiled_proto.filter_pb2,
             opencue.compiled_proto.host_pb2,
-            rqd.compiled_proto.rqd_pb2,
-            rqd.compiled_proto.host_pb2,
-            rqd.compiled_proto.report_pb2
+            opencue.compiled_proto.rqd_pb2,
+            opencue.compiled_proto.report_pb2
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.

ci/pylintrc_test

@@ -206,9 +206,8 @@ ignored-classes=optparse.Values,thread._local,_thread._local
 ignored-modules=opencue.compiled_proto,
                 opencue.compiled_proto.filter_pb2,
                 opencue.compiled_proto.host_pb2,
-                rqd.compiled_proto.rqd_pb2,
-                rqd.compiled_proto.host_pb2,
-                rqd.compiled_proto.report_pb2
+                opencue.compiled_proto.rqd_pb2,
+                opencue.compiled_proto.report_pb2
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.

ci/python_coverage_report.sh

@@ -6,17 +6,15 @@ pip install --user -r requirements.txt -r requirements_gui.txt
 
 # Protos need to have their Python code generated in order for tests to pass.
 python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
-python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
 
 # Run coverage for each component individually, but append it all into the same report.
 coverage run --source=pycue/opencue/,pycue/FileSequence/ --omit=pycue/opencue/compiled_proto/* pycue/setup.py test
 PYTHONPATH=pycue coverage run -a --source=pyoutline/outline/ pyoutline/setup.py test
 PYTHONPATH=pycue coverage run -a --source=cueadmin/cueadmin/ cueadmin/setup.py test
 PYTHONPATH=pycue xvfb-run -d coverage run -a --source=cuegui/cuegui/ cuegui/setup.py test
 PYTHONPATH=pycue:pyoutline coverage run -a --source=cuesubmit/cuesubmit/ cuesubmit/setup.py test
-coverage run -a --source=rqd/rqd/ --omit=rqd/rqd/compiled_proto/* rqd/setup.py test
+coverage run -a --source=rqd/rqd/ --omit=pycue/opencue/compiled_proto/* rqd/setup.py test
 
 # SonarCloud needs the report in XML.
 coverage xml

ci/run_python_lint.sh

@@ -9,12 +9,10 @@ pip install --user -r requirements.txt -r requirements_gui.txt
 
 # Protos need to have their Python code generated in order for tests to pass.
 python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
-python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
 
 echo "Running lint for pycue/..."
 cd pycue
@@ -49,6 +47,6 @@ cd ..
 
 echo "Running lint for rqd/..."
 cd rqd
-python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto
-python -m pylint --rcfile=../ci/pylintrc_test tests
+PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=pycue/opencue/compiled_proto
+PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests
 cd ..

ci/run_python_tests.sh

@@ -15,18 +15,16 @@ pip install --user -r requirements.txt -r requirements_gui.txt
 
 # Protos need to have their Python code generated in order for tests to pass.
 python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
-python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
 
 python pycue/setup.py test
 PYTHONPATH=pycue python pyoutline/setup.py test
 PYTHONPATH=pycue python cueadmin/setup.py test
 PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test
-python rqd/setup.py test
+PYTHONPATH=pycue python rqd/setup.py test
 
 # Xvfb no longer supports Python 2.
 if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then

ci/run_python_tests_pyside6.sh

@@ -33,11 +33,9 @@ python3 -m pip install --user PySide6==6.3.2
 
 # Protos need to have their Python code generated in order for tests to pass.
 python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
-python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
 
 # Fix compiled proto code for Python 3.
 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
 
 python pycue/setup.py test
 PYTHONPATH=pycue python pyoutline/setup.py test

cuegui/cuegui/plugins/LogViewPlugin.py

@@ -22,7 +22,6 @@
 
 from builtins import str
 from builtins import range
-import os
 import re
 import string
 import sys
@@ -33,53 +32,17 @@
 from qtpy import QtCore
 from qtpy import QtWidgets
 
+import opencue.cuelogging
 import cuegui.Constants
 import cuegui.AbstractDockWidget
 
-
 PLUGIN_NAME = 'LogView'
 PLUGIN_CATEGORY = 'Other'
 PLUGIN_DESCRIPTION = 'Displays Frame Log'
 PLUGIN_PROVIDES = 'LogViewPlugin'
 PRINTABLE = set(string.printable)
 
 
-class LogReader(object):
-    """
-    Custom class to abstract reading log files from multiple backends
-    """
-    filepath = None
-    type = None
-
-    def __init__(self, filepath):
-        """LogReader class initialization
-           @type    filepath: string
-           @param   filepath: The filepath to log to
-        """
-        self.filepath = filepath
-
-    def size(self):
-        """Return the size of the file"""
-        return int(os.stat(self.filepath).st_size)
-
-    def getMtime(self):
-        """Return modification time of the file"""
-        return os.path.getmtime(self.filepath)
-
-    def exists(self):
-        """Check if the file exists"""
-        return os.path.exists(self.filepath)
-
-    def read(self):
-        """Read the data from the backend"""
-        content = None
-        if self.exists() is True:
-            with open(self.filepath, "r", encoding='utf-8') as fp:
-                content = fp.read()
-
-        return content
-
-
 class LineNumberArea(QtWidgets.QWidget):
     """
     Custom widget for the line numbers. This widget is designed to be attached
@@ -849,7 +812,7 @@ def _display_log_content(self):
         @postcondition: The _update_log method is scheduled to run again
                         after 5 seconds
         """
-        log_reader = LogReader(self._log_file)
+        log_reader = opencue.cuelogging.CueLogger(self._log_file, mode=opencue.cuelogging.MODE_READ)
 
         try:
             if log_reader.exists() is not True:

proto/README.md

@@ -8,24 +8,6 @@ To use them, they must first be compiled into the native language of the compone
 
 Gradle automatically compiles these proto files, no further action is needed.
 
-## RQD
-
-To generate:
-
-```sh
-python -m grpc_tools.protoc -I=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto ./*.proto
-2to3 -wn -f import ../rqd/rqd/compiled_proto/*_pb2*.py
-```
-
-For Windows (Powershell):
-
-```powershell
-python -m grpc_tools.protoc --proto_path=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto (ls *.proto).Name
-cd ..\rqd\rqd\compiled_proto\
-2to3 -wn -f import (ls *_pb2*.py).Name
-```
-
-
 ## pycue
 
 To generate:

pycue/opencue/cuelogging.py

@@ -0,0 +1,166 @@
+#  Copyright Contributors to the OpenCue Project
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+"""Module for reading and writing log files"""
+
+import logging
+import os
+import platform
+import datetime
+import time
+
+log = logging.getLogger(__name__)
+
+MODE_READ = 0
+MODE_WRITE = 1
+
+class CueLogger(object):
+    """Class to abstract file logging, this class tries to act as a file object"""
+    filepath = None
+    fd = None
+    mode = MODE_READ  # Default in read mode
+
+    def __init__(self, filepath, mode, maxLogFiles=1):
+        """RQDLogger class initialization
+           @type    filepath: string
+           @param   filepath: The filepath to log to
+           @type    mode: int
+           @param   mode: read or write mode
+           @type    maxLogFiles: int
+           @param   maxLogFiles: number of files to rotate, when in write mode
+        """
+
+        self.filepath = filepath
+        self.mode = mode
+        if self.mode == MODE_WRITE:
+            log_dir = os.path.dirname(self.filepath)
+            if not os.access(log_dir, os.F_OK):
+                # Attempting mkdir for missing logdir
+                msg = "No Error"
+                try:
+                    os.makedirs(log_dir)
+                    os.chmod(log_dir, 0o777)
+                # pylint: disable=broad-except
+                except Exception as e:
+                    # This is expected to fail when called in abq
+                    # But the directory should now be visible
+                    msg = e
+
+                if not os.access(log_dir, os.F_OK):
+                    err = "Unable to see log directory: %s, mkdir failed with: %s" % (
+                        log_dir, msg)
+                    raise RuntimeError(err)
+
+            if not os.access(log_dir, os.W_OK):
+                err = "Unable to write to log directory %s" % log_dir
+                raise RuntimeError(err)
+
+            try:
+                # Rotate any old logs to a max of MAX_LOG_FILES:
+                if os.path.isfile(self.filepath):
+                    rotateCount = 1
+                    while (os.path.isfile("%s.%s" % (self.filepath, rotateCount))
+                           and rotateCount < maxLogFiles):
+                        rotateCount += 1
+                    os.rename(self.filepath,
+                              "%s.%s" % (self.filepath, rotateCount))
+            # pylint: disable=broad-except
+            except Exception as e:
+                err = "Unable to rotate previous log file due to %s" % e
+                # Windows might fail while trying to rotate logs for checking if file is
+                # being used by another process. Frame execution doesn't need to
+                # be halted for this.
+                if platform.system() == "Windows":
+                    log.warning(err)
+                else:
+                    raise RuntimeError(err)
+            # pylint: disable=consider-using-with
+            self.fd = open(self.filepath, "w+", 1, encoding='utf-8')
+            try:
+                os.chmod(self.filepath, 0o666)
+            # pylint: disable=broad-except
+            except Exception as e:
+                err = "Failed to chmod log file! %s due to %s" % (self.filepath, e)
+                log.warning(err)
+        elif mode == MODE_READ:
+            self.fd = open(self.filepath, "r", encoding='utf-8')
+        else:
+            log.error("Unknown mode for log file!")
+
+    # pylint: disable=arguments-differ
+    def write(self, data, prependTimestamp=False):
+        """Abstract write function that will write to the correct backend"""
+        # Only work if in write mode, otherwise ignore
+        if self.mode == MODE_WRITE:
+            # Convert data to unicode
+            if isinstance(data, bytes):
+                data = data.decode('utf-8', errors='ignore')
+            if prependTimestamp is True:
+                lines = data.splitlines()
+                curr_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S")
+                for line in lines:
+                    print("[%s] %s" % (curr_line_timestamp, line), file=self)
+            else:
+                self.fd.write(data)
+
+    def writelines(self, __lines):
+        """Provides support for writing mutliple lines at a time"""
+        for line in __lines:
+            self.write(line)
+
+    def close(self):
+        """Closes the file if the backend is file based"""
+        self.fd.close()
+
+    def waitForFile(self, maxTries=5):
+        """Waits for the file to exist before continuing when using a file backend"""
+        # Waits for a file to exist
+        tries = 0
+        while tries < maxTries:
+            if os.path.exists(self.filepath):
+                return
+            tries += 1
+            time.sleep(0.5 * tries)
+        raise IOError("Failed to create %s" % self.filepath)
+
+    def size(self):
+        """Return the size of the file"""
+        return int(os.stat(self.filepath).st_size)
+
+    def getMtime(self):
+        """Return modification time of the file"""
+        return os.path.getmtime(self.filepath)
+
+    def exists(self):
+        """Check if the file exists"""
+        return os.path.exists(self.filepath)
+
+    def read(self):
+        """Read the data from the backend"""
+        # Only allow reading when in read mode
+
+        content = None
+        if self.mode == MODE_READ:
+            if self.exists() is True:
+                with open(self.filepath, "r", encoding='utf-8') as fp:
+                    content = fp.read()
+
+        return content
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass

rqd/Dockerfile

@@ -27,16 +27,6 @@ COPY rqd/setup.py ./rqd/
 COPY rqd/tests/ ./rqd/tests
 COPY rqd/rqd/ ./rqd/rqd
 
-RUN python3.9 -m grpc_tools.protoc \
-  -I=./proto \
-  --python_out=./rqd/rqd/compiled_proto \
-  --grpc_python_out=./rqd/rqd/compiled_proto \
-  ./proto/*.proto
-
-# Fix imports to work in both Python 2 and 3. See
-# <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
-
 COPY VERSION.in VERSIO[N] ./
 RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION