VDMS 2.2 Release (#142) (#143)

* Flexible Retention Policy (Auto-deletion): Users can automatically have node removed from VDMS

* Internal Database Retry: Retry query before asserting and exiting database 

* Allow User-defined Operators in VCL: User can define a new operator in VCL and expose it in the VDMS API.  Provided a sample custom operation 

* Improve Descriptor Search: Fixed descriptor bug (insert multiple descriptors) and added the ability to search descriptors by query blob AND constraints

* JPEG Storage Quality: Image differences due to lossy compression of JPEG. Added disclaimer to user to use lossless format (PNG) for inference applications (https://github.com/IntelLabs/vdms/wiki/AddImage#inference-image-format )

* Increase Buffer Size:   We increased the max buffer size for database transactions to 1GB and set a default size of 32MB (previous max)

* Removed Dockerfiles: Only provide base Dockerfile

.gitignore

@@ -1,3 +1,7 @@
 build/
 db/
-.vscode
+*.gcov
+
+# VS Code Specific
+.devcontainer
+.vscode

.gitmodules

@@ -2,3 +2,4 @@
 	path = src/pmgd
 	url = https://github.com/omp87/pmgd.git
 	branch = develop
+

CMakeLists.txt

@@ -1,4 +1,11 @@
 cmake_minimum_required (VERSION 3.17)
+
+IF(CODE_COVERAGE)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
+    enable_testing()
+ENDIF()
+
 project(vdms_application)
 add_compile_options(-g -fPIC)
 
@@ -9,25 +16,19 @@ include_directories(${Protobuf_INCLUDE_DIRS})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 execute_process(COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json ${CMAKE_CURRENT_BINARY_DIR}/APISchema.h)
-
-message(STATUS "before " ${PROTO_SRCS} ${PROTO_HDRS})
 protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS utils/src/protobuf/partitionerMessages.proto utils/src/protobuf/pmgdMessages.proto utils/src/protobuf/queryMessage.proto)
 add_library(vdms_protobuf SHARED ${PROTO_SRCS} ${PROTO_HDRS})
-message(STATUS "after " ${PROTO_SRCS} ${PROTO_HDRS})
 
 add_subdirectory(src/pmgd)
 add_subdirectory(utils)
 add_subdirectory(tests)
 add_subdirectory(src/vcl)
 add_subdirectory(client/cpp)
+add_subdirectory(ext/custom_vcl)
 link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/)
 include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf)
-add_library(dms SHARED src/BoundingBoxCommand.cc src/CommunicationManager.cc src/DescriptorsCommand.cc src/DescriptorsManager.cc src/ExceptionsCommand.cc src/ImageCommand.cc src/PMGDIterators.cc src/PMGDQuery.cc src/PMGDQueryHandler.cc src/QueryHandler.cc src/QueryMessage.cc src/RSCommand.cc src/SearchExpression.cc src/Server.cc src/VDMSConfig.cc src/VideoCommand.cc ${PROTO_SRCS} ${PROTO_HDRS})
-target_link_libraries(dms vcl pmgd pmgd-util protobuf vdms-utils)
+add_library(dms SHARED src/BoundingBoxCommand.cc src/CommunicationManager.cc src/DescriptorsCommand.cc src/DescriptorsManager.cc src/ExceptionsCommand.cc src/ImageCommand.cc src/PMGDIterators.cc src/PMGDQuery.cc src/PMGDQueryHandler.cc src/QueryHandler.cc src/QueryMessage.cc src/RSCommand.cc src/SearchExpression.cc src/Server.cc src/VDMSConfig.cc src/VideoCommand.cc src/AutoDeleteNode.cc ${PROTO_SRCS} ${PROTO_HDRS})
+target_link_libraries(dms vcl pmgd pmgd-util protobuf vdms-utils pthread)
 
 add_executable(vdms src/vdms.cc)
-target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss jsoncpp ${OpenCV_LIBS})
-
-#https://cmake.org/cmake/help/v3.9/module/FindProtobuf.html
-
-#target_link_libraries(bar ${Protobuf_LIBRARIES})
+target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss jsoncpp ${OpenCV_LIBS})

INSTALL.md

@@ -44,7 +44,7 @@ Here is the detailed process of installation of VDMS dependencies.
 ### Install CMAKE 
     cd CMake && ./bootstrap && make -j16 && make install && \\
     cd / && cd swig && \ 
-    ./autogen.sh && ./configure && make -j16 && make install 
+    ./autogen.sh && ./configure && make -j16 && make install
 ### Install Faiss
     cd faiss && mkdir build && cd build && \
     cmake -DFAISS_ENABLE_GPU=OFF .. && \\
@@ -147,12 +147,12 @@ Build TileDB
     sudo make install-tiledb
 
 ### [Maven]
-    sudo ln -s /grpc/third_party/protobuf/cmake/build/protoc 
+    sudo ln -s /grpc/third_party/protobuf/cmake/build/protoc
     grpc/third_party/protobuf/src/protoc && \
     sudo apt-get install -y maven && \
     cd /grpc/third_party/protobuf/java/core &&\
     mvn package &&\
-    cp target/protobuf-java-3.13.0.jar /usr/share/java/protobuf.jar &&\
+    cp target/protobuf-java-3.13.0.jar /usr/share/java/protobuf.jar && \
     curl https://storage.googleapis.com/google-code-archive- 
     downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar -o /usr/share/java/json- 
     simple-1.1.1.jar && cd / &&\
@@ -189,7 +189,7 @@ You may need to change proxy setting for Maven if you are behind a proxy like th
     cd vdms && git checkout develop &&\
     git submodule update --init --recursive && \
     cd vdms && mkdir build && cd build && cmake .. && make -j16 && \
-    cp ../config-vdms.json . 
+    cp ../config-vdms.json .
 
 
 

docker/README.md

@@ -4,12 +4,12 @@ Here are the instructions to build a docker image of the VDMS Server.
 
 The build command is: 
 
-    cd server
+    cd base
     Dockerfile | docker build -t vdms:latest .
 
 If you are under a proxy, use:
 
-    cd server
+    cd base
     Dockerfile | docker build -t vdms:latest --build-arg=http_proxy --build-arg=https_proxy .
 
 To run the docker image as a container, include the --net flag.

docker/base/Dockerfile

@@ -4,6 +4,7 @@
 ARG UBUNTU_VERSION=20.04
 ARG UBUNTU_NAME=focal
 ARG BUILD_THREADS=-j16
+ARG MAVEN_OPTS='-Dhttps.nonProxyHosts="localhost|127.0.0.1"'
 
 #1
 FROM ubuntu:${UBUNTU_VERSION}
@@ -15,15 +16,15 @@ ARG MAVEN_OPTS
 
 #Install Packages
 RUN apt-get update && apt-get -y install software-properties-common && \
-add-apt-repository "deb http://security.ubuntu.com/ubuntu xenial-security main" && \
+add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main" && \
 apt-get -y install g++ git libssl-dev libc-ares-dev apt-transport-https \
 ca-certificates curl gnupg-agent software-properties-common cmake python3-pip \
 build-essential autoconf automake libtool g++ unzip bzip2 libarchive-tools \
 cmake git pkg-config python python-dev wget libbz2-dev libssl-dev liblz4-dev \
 mpich libjsoncpp-dev flex javacc bison openjdk-11-jdk-headless libleveldb-dev \
 libsnappy-dev libhdf5-serial-dev libatlas-base-dev libboost-all-dev \
 libgflags-dev libgoogle-glog-dev liblmdb-dev libjpeg8-dev libtiff5-dev \
-libjasper-dev libgtk-3-dev libopenmpi-dev libgtest-dev ed libgtk2.0-dev \
+libgtk-3-dev libopenmpi-dev libgtest-dev ed libgtk2.0-dev \
 pkg-config libavcodec-dev libavformat-dev libswscale-dev libtbb2 libtbb-dev \
 libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev libopenblas-dev maven && \
 pip3 install numpy
@@ -82,10 +83,9 @@ RUN apt-get install -y maven
 
 RUN ln -s /grpc/third_party/protobuf/cmake/build/protoc \
     /grpc/third_party/protobuf/src/protoc && \
-    cd /grpc/third_party/protobuf/java/core && mvn package && \
-    echo "cp $(ls target/protobuf-java*.jar) /usr/share/java/protobuf.jar" > \
-    tmp_script && bash tmp_script && \
-    cd / 
+    cd /grpc/third_party/protobuf/java/core && \
+    mvn package && \
+    cp $(ls target/protobuf-java*.jar) /usr/share/java/protobuf.jar
 
 # Valijson
 RUN cd /valijson && cp -r include/* /usr/local/include/