From 6c394f673e07a12787ba70a4ee7c75277f16e312 Mon Sep 17 00:00:00 2001
From: Andrew Kent <andrew@galois.com>
Date: Fri, 9 Oct 2020 15:11:08 -0700
Subject: [PATCH 1/3] Add simple example argo server with unit tests.

---
 cabal.project                                 |   1 +
 file-echo-api/CHANGELOG.md                    |   7 +
 file-echo-api/file-echo-api.cabal             |  71 ++++++++++
 file-echo-api/file-echo-api/Main.hs           |  33 +++++
 file-echo-api/src/FileEchoServer.hs           | 120 ++++++++++++++++
 file-echo-api/test-scripts/file-echo-tests.py | 132 ++++++++++++++++++
 .../test-scripts/test-data/hello.txt          |   1 +
 file-echo-api/test/Test.hs                    |  30 ++++
 8 files changed, 395 insertions(+)
 create mode 100644 file-echo-api/CHANGELOG.md
 create mode 100644 file-echo-api/file-echo-api.cabal
 create mode 100644 file-echo-api/file-echo-api/Main.hs
 create mode 100644 file-echo-api/src/FileEchoServer.hs
 create mode 100644 file-echo-api/test-scripts/file-echo-tests.py
 create mode 100644 file-echo-api/test-scripts/test-data/hello.txt
 create mode 100644 file-echo-api/test/Test.hs

diff --git a/cabal.project b/cabal.project
index 9c9044d..211613c 100644
--- a/cabal.project
+++ b/cabal.project
@@ -1,6 +1,7 @@
 packages:
   argo/
   python/
+  file-echo-api/
   cryptol-remote-api/
   saw-remote-api/
   tasty-script-exitcode/
diff --git a/file-echo-api/CHANGELOG.md b/file-echo-api/CHANGELOG.md
new file mode 100644
index 0000000..a8487c8
--- /dev/null
+++ b/file-echo-api/CHANGELOG.md
@@ -0,0 +1,7 @@
+# Revision history for file-echo-api
+
+## 0.1.0.0 -- 2020-10-09
+
+* First version. Released on an unsuspecting world. A simple echo server which
+  can load files on disk and send their contents (all or a portion) back
+  to the client.
diff --git a/file-echo-api/file-echo-api.cabal b/file-echo-api/file-echo-api.cabal
new file mode 100644
index 0000000..9699efa
--- /dev/null
+++ b/file-echo-api/file-echo-api.cabal
@@ -0,0 +1,71 @@
+cabal-version:       2.4
+name:                file-echo-api
+version:             0.1.0.0
+license:             BSD-3-Clause
+license-file:        LICENSE
+author:              Andrew Kent
+maintainer:          andrew@galois.com
+category:            Language
+extra-source-files:  CHANGELOG.md
+data-files:          test-scripts/**/*.py
+                     test-scripts/**/*.txt
+
+common warnings
+  ghc-options:
+    -Weverything
+    -Wno-missing-exported-signatures
+    -Wno-missing-import-lists
+    -Wno-missed-specialisations
+    -Wno-all-missed-specialisations
+    -Wno-unsafe
+    -Wno-safe
+    -Wno-missing-local-signatures
+    -Wno-monomorphism-restriction
+    -Wno-implicit-prelude
+
+common deps
+  build-depends:
+    base                 >=4.11.1.0 && <4.15,
+    argo,
+    aeson                >= 1.4.2,
+    base64-bytestring    >= 1.0,
+    bytestring           ^>= 0.10.8,
+    containers           >=0.5.11 && <0.7,
+    directory            ^>= 1.3.1,
+    filepath             ^>= 1.4,
+    lens                 >= 4.17 && < 4.20,
+    scientific           ^>= 0.3,
+    text                 ^>= 1.2.3,
+    unordered-containers ^>= 0.2,
+    vector               ^>= 0.12,
+
+  default-language:    Haskell2010
+
+library
+  import:              deps, warnings
+  hs-source-dirs:      src
+
+  exposed-modules:
+    FileEchoServer
+
+executable file-echo-api
+  import:              deps, warnings
+  main-is:             Main.hs
+  hs-source-dirs:      file-echo-api
+
+  build-depends:
+    file-echo-api
+
+test-suite test-file-echo-api
+  import:              deps, warnings
+  type:                exitcode-stdio-1.0
+  hs-source-dirs:      test
+  main-is:             Test.hs
+  other-modules:       Paths_file_echo_api
+  build-depends:
+    argo-python,
+    file-echo-api,
+    quickcheck-instances ^>= 0.3.19,
+    tasty                 >= 1.2.1,
+    tasty-quickcheck     ^>= 0.10,
+    tasty-script-exitcode
diff --git a/file-echo-api/file-echo-api/Main.hs b/file-echo-api/file-echo-api/Main.hs
new file mode 100644
index 0000000..b701228
--- /dev/null
+++ b/file-echo-api/file-echo-api/Main.hs
@@ -0,0 +1,33 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE PartialTypeSignatures #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+module Main ( main ) where
+
+import qualified Data.Aeson as JSON
+import           Data.ByteString ( ByteString )
+import Data.Text (Text)
+
+import qualified Argo as Argo
+import Argo.DefaultMain ( defaultMain )
+
+
+import qualified FileEchoServer as FES
+
+main :: IO ()
+main =
+  do theApp <- Argo.mkApp mkInitState serverMethods
+     defaultMain description theApp
+
+description :: String
+description =
+  "An RPC server for loading and printing files."
+
+mkInitState :: (FilePath -> IO ByteString) -> IO FES.ServerState 
+mkInitState = const $ FES.initialState
+
+serverMethods :: [(Text, Argo.MethodType, JSON.Value -> Argo.Method FES.ServerState JSON.Value)]
+serverMethods =
+  [ ("load",           Argo.Command,   Argo.method FES.loadCmd)
+  , ("clear",          Argo.Command,   Argo.method FES.clearCmd)
+  , ("show",           Argo.Query,     Argo.method FES.showCmd)
+  ]
diff --git a/file-echo-api/src/FileEchoServer.hs b/file-echo-api/src/FileEchoServer.hs
new file mode 100644
index 0000000..7118e70
--- /dev/null
+++ b/file-echo-api/src/FileEchoServer.hs
@@ -0,0 +1,120 @@
+{-# LANGUAGE OverloadedStrings #-}
+module FileEchoServer ( module FileEchoServer ) where
+
+import qualified Argo as Argo
+import           Control.Monad.IO.Class ( liftIO )
+import qualified Data.Aeson as JSON
+import           Data.Aeson ( (.:), (.:?), (.=), (.!=) )
+import           Data.ByteString ( ByteString )
+import qualified Data.ByteString.Char8 as Char8
+import qualified Data.Text as T
+import qualified System.Directory as Dir
+
+newtype FileContents = FileContents String
+
+data ServerState = ServerState 
+  { loadedFile :: Maybe FilePath
+  -- ^ Loaded file (if any).
+  , fileContents :: FileContents
+  -- ^ Current file contents, or "" if one has not been loaded yet.
+  }
+
+initialState :: IO ServerState
+initialState = pure $ ServerState Nothing (FileContents "")
+
+newtype ServerErr = ServerErr String
+newtype ServerRes a = ServerRes (Either ServerErr (a,FileContents))
+newtype ServerCmd a = 
+  ServerCmd (((FilePath -> IO ByteString), FileContents) -> IO (ServerRes a))
+
+
+------------------------------------------------------------------------
+-- Command Execution
+
+runServerCmd :: ServerCmd a -> Argo.Method ServerState a
+runServerCmd (ServerCmd cmd) =
+    do s <- Argo.getState
+       reader <- Argo.getFileReader
+       out <- liftIO $ cmd (reader, fileContents s)
+       case out of
+         ServerRes (Left (ServerErr message)) ->
+           Argo.raise $ Argo.makeJSONRPCException
+            11000 "File Server exception"
+            (Just (JSON.object ["error" .= message]))
+         ServerRes (Right (x, newFileContents)) ->
+           do Argo.setState (s { fileContents = newFileContents})
+              return x
+
+
+------------------------------------------------------------------------
+-- Errors
+
+fileNotFound :: FilePath -> Argo.JSONRPCException
+fileNotFound fp =
+  Argo.makeJSONRPCException
+    20051 (T.pack ("File doesn't exist: " <> fp))
+    (Just (JSON.object ["path" .= fp]))
+
+------------------------------------------------------------------------
+-- Load Command
+
+data LoadParams = LoadParams FilePath
+
+instance JSON.FromJSON LoadParams where
+  parseJSON =
+    JSON.withObject "params for \"load\"" $
+    \o -> LoadParams <$> o .: "file path"
+
+loadCmd :: LoadParams -> Argo.Method ServerState ()
+loadCmd (LoadParams file) =
+  do exists <- liftIO $ Dir.doesFileExist file
+     if exists
+     then do getFileContents <- Argo.getFileReader
+             contents <- liftIO $ getFileContents file
+             Argo.setState $ ServerState 
+              { loadedFile = Just file
+              , fileContents = FileContents $ Char8.unpack contents
+              }
+     else Argo.raise (fileNotFound file)
+
+
+------------------------------------------------------------------------
+-- Clear Command
+
+data ClearParams = ClearParams
+
+instance JSON.FromJSON ClearParams where
+  parseJSON =
+    JSON.withObject "params for \"show\"" $
+    \o -> pure ClearParams
+
+clearCmd :: ClearParams -> Argo.Method ServerState ()
+clearCmd _ =
+  do Argo.setState $ ServerState
+      { loadedFile = Nothing
+      , fileContents = FileContents ""
+      }
+
+-- Substring ------------------------------------------------------------
+
+data ShowParams = ShowParams
+  { showStart :: Int
+    -- ^ Inclusive start index in contents.
+  , showEnd :: Maybe Int
+  -- ^ Exclusive end index in contents.
+  }
+
+instance JSON.FromJSON ShowParams where
+  parseJSON =
+    JSON.withObject "params for \"show\"" $
+    \o -> do start <- o .:? "start" .!= 0
+             end <- o   .:? "end"
+             pure $ ShowParams start end
+
+showCmd :: ShowParams -> Argo.Method ServerState JSON.Value
+showCmd (ShowParams start end) =
+  do (FileContents contents) <-  fileContents <$> Argo.getState
+     let len = case end of
+                Nothing -> length contents
+                Just idx -> idx - start 
+     pure (JSON.object [ "value" .= (JSON.String $ T.pack $ take len $ drop start contents)])
diff --git a/file-echo-api/test-scripts/file-echo-tests.py b/file-echo-api/test-scripts/file-echo-tests.py
new file mode 100644
index 0000000..ce38322
--- /dev/null
+++ b/file-echo-api/test-scripts/file-echo-tests.py
@@ -0,0 +1,132 @@
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import time
+
+import argo.connection as argo
+
+dir_path = Path(os.path.dirname(os.path.realpath(__file__)))
+
+file_dir = dir_path.joinpath('test-data')
+
+if not file_dir.is_dir():
+    print('ERROR: ' + str(file_dir) + ' is not a directory!')
+    assert(False)
+
+
+def run_tests(c):
+    ## Positive tests -- make sure the server behaves as we expect with valid RPCs
+
+    # Check that their is nothing to show if we haven't loaded a file yet
+    uid = c.send_message("show", {"state": None})
+    actual = c.wait_for_reply_to(uid)
+    expected = {'result':{'state':None,'stdout':'','stderr':'','answer':{'value':''}},'jsonrpc':'2.0','id':uid}
+    assert(actual == expected)
+    prev_uid = uid
+
+    # load a file
+    hello_file = file_dir.joinpath('hello.txt')
+    assert(False if not hello_file.is_file() else True)
+    uid = c.send_message("load", {"file path": str(hello_file), "state": None})
+    actual = c.wait_for_reply_to(uid)
+    assert('result' in actual and 'state' in actual['result'])
+    hello_state = actual['result']['state']
+    expected = {'result':{'state':hello_state,'stdout':'','stderr':'','answer':[]},'jsonrpc':'2.0','id':uid}
+    assert(actual == expected)
+    assert(uid != prev_uid)
+    assert(hello_state != None)
+    prev_uid = uid
+
+    # check the contents of the loaded file
+    uid = c.send_message("show", {"state": hello_state})
+    actual = c.wait_for_reply_to(uid)
+    expected = {'result':{'state':hello_state,'stdout':'','stderr':'','answer':{'value':'Hello World!\n'}},'jsonrpc':'2.0','id':uid}
+    assert(actual == expected)
+    assert(uid != prev_uid)
+    prev_uid = uid
+
+    # check a _portion_ of the contents of the loaded file
+    uid = c.send_message("show", {"start":1, "end":5, "state": hello_state})
+    actual = c.wait_for_reply_to(uid)
+    expected = {'result':{'state':hello_state,'stdout':'','stderr':'','answer':{'value':'ello'}},'jsonrpc':'2.0','id':uid}
+    assert(actual == expected)
+    assert(uid != prev_uid)
+    prev_uid = uid
+
+    # clear the loaded file
+    uid = c.send_message("clear", {"state": hello_state})
+    actual = c.wait_for_reply_to(uid)
+    assert('result' in actual and 'state' in actual['result'])
+    cleared_state = actual['result']['state']
+    expected = {'result':{'state':cleared_state,'stdout':'','stderr':'','answer':[]},'jsonrpc':'2.0','id':uid}
+    assert(actual == expected)
+    assert(cleared_state != hello_state)
+    assert(uid != prev_uid)
+
+    # check that the file contents cleared
+    uid = c.send_message("show", {"state": cleared_state})
+    actual = c.wait_for_reply_to(uid)
+    expected = {'result':{'state':cleared_state,'stdout':'','stderr':'','answer':{'value':''}},'jsonrpc':'2.0','id':uid}
+    assert(actual == expected)
+
+    ## Negative tests -- make sure the server errors as we expect
+
+    # Method not found
+    uid = c.send_message("bad function", {"state": cleared_state})
+    actual = c.wait_for_reply_to(uid)
+    expected = {'error':{'data':{'stdout':'','data':'bad function','stderr':''},'code':-32601,'message':'Method not found'},'jsonrpc':'2.0','id':uid}
+    assert(actual == expected)
+
+    # Invalid params
+    uid = c.send_message("load", {"file path": 12345, "state": cleared_state})
+    actual = c.wait_for_reply_to(uid)
+    expected = {'error':{'data':{'stdout':'','data':{'state':cleared_state,'file path':12345},'stderr':''},'code':-32602,'message':'Invalid params: expected String, but encountered Number'},'jsonrpc':'2.0','id':uid}
+    assert(actual == expected)
+
+    # load a nonexistent file, check error that is returned
+    nonexistent_file = file_dir.joinpath('nonexistent.txt')
+    if nonexistent_file.is_file():
+        print('ERROR: ' + str(nonexistent_file) + ' was expected to not exist, but it does!')
+        assert(False)
+    uid = c.send_message("load", {"file path": str(nonexistent_file), "state": cleared_state})
+    actual = c.wait_for_reply_to(uid)
+    expected = {'error':{'data':{'stdout':'','data':{'path':str(nonexistent_file)},'stderr':''},'code':20051,'message':'File doesn\'t exist: ' + str(nonexistent_file)},'jsonrpc':'2.0','id':uid}
+    assert(actual == expected)
+
+# Test with both sockets and stdio
+env = os.environ.copy()
+
+# Launch a separate process for the RemoteSocketProcess test
+p = subprocess.Popen(
+    ["cabal", "v2-exec", "file-echo-api", "--verbose=0", "--", "--port", "50005"],
+    stdout=subprocess.DEVNULL,
+    stdin=subprocess.DEVNULL,
+    stderr=subprocess.DEVNULL,
+    start_new_session=True,
+    env=env)
+
+time.sleep(5)
+assert(p is not None)
+assert(p.poll() is None)
+
+# Test argo's RemoteSocketProcess
+c = argo.ServerConnection(
+       argo.RemoteSocketProcess('localhost', 50005, ipv6=True))
+
+run_tests(c)
+
+# close the remote process, we don't need it for the remaining tests
+os.killpg(os.getpgid(p.pid), signal.SIGKILL)
+
+# Test argo's DynamicSocketProcess
+c = argo.ServerConnection(
+       argo.DynamicSocketProcess("cabal v2-exec file-echo-api --verbose=0 -- --port 50005"))
+
+run_tests(c)
+
+c = argo.ServerConnection(
+       argo.StdIOProcess("cabal v2-exec file-echo-api --verbose=0 -- --stdio"))
+
+run_tests(c)
diff --git a/file-echo-api/test-scripts/test-data/hello.txt b/file-echo-api/test-scripts/test-data/hello.txt
new file mode 100644
index 0000000..980a0d5
--- /dev/null
+++ b/file-echo-api/test-scripts/test-data/hello.txt
@@ -0,0 +1 @@
+Hello World!
diff --git a/file-echo-api/test/Test.hs b/file-echo-api/test/Test.hs
new file mode 100644
index 0000000..60c0a06
--- /dev/null
+++ b/file-echo-api/test/Test.hs
@@ -0,0 +1,30 @@
+
+module Main ( module Main ) where
+
+import Test.Tasty
+import Test.Tasty.HUnit.ScriptExit
+
+
+import Argo.PythonBindings
+import Paths_file_echo_api
+
+import FileEchoServer()
+-- ^ We import FileEchoServer to force rebuild when building
+-- the tests in case changes have been made to server.
+
+main :: IO ()
+main =
+  do reqs <- getArgoPythonFile "requirements.txt"
+     withPython3venv (Just reqs) $ \pip python ->
+       do pySrc <- getArgoPythonFile "."
+          testScriptsDir <- getDataFileName "test-scripts/"
+          --let testScriptsDir = "/Users/andrew/Repos/GRINDSTONE/argo/file-echo-api/test-scripts"
+          pip ["install", pySrc]
+          putStrLn "pipped"
+
+          scriptTests <- makeScriptTests testScriptsDir [python]
+
+          defaultMain $
+            testGroup "Tests for file-echo-api"
+              [ testGroup "Scripting API tests" scriptTests
+              ]

From 96205ff8452b1ca0acee4372e91fcc18be3a7e03 Mon Sep 17 00:00:00 2001
From: Andrew Kent <andrew@galois.com>
Date: Fri, 9 Oct 2020 15:27:22 -0700
Subject: [PATCH 2/3] add README.md for file-echo-api

---
 file-echo-api/README.md | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 file-echo-api/README.md

diff --git a/file-echo-api/README.md b/file-echo-api/README.md
new file mode 100644
index 0000000..1b9ec44
--- /dev/null
+++ b/file-echo-api/README.md
@@ -0,0 +1,28 @@
+# file-echo-api
+
+A simple example usage of Argo: a JSON-RPC "file echo server" which can load files on disk and send their contents back to the client.
+
+## file-echo-api commands
+
++ `load`, loads a file into the echo server's memory
+  - Parameters
+    * `file path` : file path as a string, describing which file to load
+
++ `clear`, clears any loaded file from the server's memory
+  - No parameters
+
+## file-echo-api queries
+
++ `show`, returns the contents of the last loaded file
+  - No Required Parameters
+  - Optional Parameters
+    * `start` : character index in loaded file to begin showing from, default value `0`
+    * `end` : character index to show up until (but not including), default value is the character length of the currently loaded file minus the `start` parameter
+
+
+# Files
+
++ `src/FileEchoServer.hs` implements the internals of the server
++ `file-echo-api/Main.hs` defines an Argo executable leveraging the definitions from `FileEchoServer.hs`
++ `test/Test.hs` a Haskell test runner which executes the python script `file-echo-tests.py`
++ `test-scripts/file-echo-tests.py` is a python script which leverages the `argo` python library to test the file echo server

From c46f8562dc3529c13b42ef5283c2fd942789ef9e Mon Sep 17 00:00:00 2001
From: Andrew Kent <andrew@galois.com>
Date: Mon, 12 Oct 2020 11:19:16 -0700
Subject: [PATCH 3/3] doc: file-echo-api/README.md => file-echo-api/README.rst

---
 file-echo-api/README.md  | 28 -----------------------
 file-echo-api/README.rst | 48 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 28 deletions(-)
 delete mode 100644 file-echo-api/README.md
 create mode 100644 file-echo-api/README.rst

diff --git a/file-echo-api/README.md b/file-echo-api/README.md
deleted file mode 100644
index 1b9ec44..0000000
--- a/file-echo-api/README.md
+++ /dev/null
@@ -1,28 +0,0 @@
-# file-echo-api
-
-A simple example usage of Argo: a JSON-RPC "file echo server" which can load files on disk and send their contents back to the client.
-
-## file-echo-api commands
-
-+ `load`, loads a file into the echo server's memory
-  - Parameters
-    * `file path` : file path as a string, describing which file to load
-
-+ `clear`, clears any loaded file from the server's memory
-  - No parameters
-
-## file-echo-api queries
-
-+ `show`, returns the contents of the last loaded file
-  - No Required Parameters
-  - Optional Parameters
-    * `start` : character index in loaded file to begin showing from, default value `0`
-    * `end` : character index to show up until (but not including), default value is the character length of the currently loaded file minus the `start` parameter
-
-
-# Files
-
-+ `src/FileEchoServer.hs` implements the internals of the server
-+ `file-echo-api/Main.hs` defines an Argo executable leveraging the definitions from `FileEchoServer.hs`
-+ `test/Test.hs` a Haskell test runner which executes the python script `file-echo-tests.py`
-+ `test-scripts/file-echo-tests.py` is a python script which leverages the `argo` python library to test the file echo server
diff --git a/file-echo-api/README.rst b/file-echo-api/README.rst
new file mode 100644
index 0000000..fa48376
--- /dev/null
+++ b/file-echo-api/README.rst
@@ -0,0 +1,48 @@
+``file-echo-api``
+=================
+
+A simple example usage of Argo: a JSON-RPC "file echo server" which can load files on disk and send their contents back to the client.
+
+Commands
+--------------------------
+
+* ``load``
+
+  * Loads a file into the echo server's memory.  
+  * Parameters
+
+    * ``file path : String``
+  
+      * File path describing which file to load.
+
+* ``clear``
+  
+  * Clears any loaded file from the server's memory.
+  * No parameters
+
+Queries
+-------------------------
+
+* ``show``
+  
+  * Returns the contents of the last loaded file.
+
+  * No required parameters
+  * Optional parameters
+
+    * ``start : Integer``
+  
+      * Character index in loaded file to begin showing from, default value ``0``.
+  
+    * ``end : Integer``
+  
+      * Character index to show up until (but not including), default value is the character length of the currently loaded file minus the ``start`` parameter's value.
+
+
+Files
+-----
+
+* ``src/FileEchoServer.hs`` implements the internals of the server.
+* ``file-echo-api/Main.hs`` defines an Argo executable leveraging the definitions from ``FileEchoServer.hs``.
+* ``test/Test.hs`` a Haskell test runner which executes the python script ``file-echo-tests.py``.
+* ``test-scripts/file-echo-tests.py`` is a python script which leverages the ``argo`` python library to test the file echo server.