[feat] Add Windows support

aim/cli/convert/processors/tensorboard.py

@@ -44,6 +44,7 @@ def get_parent(current_path, level=0):
         # level 0 is the direct parent directory
         if level <= 0:
             return os.path.dirname(current_path)
+        # FIXME: probably broken on Windows (C:\)
         elif current_path in ('', '.', '/'):
             return current_path
         return get_parent(os.path.dirname(current_path), level - 1)

aim/sdk/lock_manager.py

@@ -11,10 +11,10 @@
 from pathlib import Path
 from dataclasses import dataclass, field
 from dateutil.relativedelta import relativedelta
-from filelock import UnixFileLock, SoftFileLock, Timeout
+from filelock import SoftFileLock, Timeout
 
 from aim.sdk.errors import RunLockingError
-from aim.storage.locking import RunLock
+from aim.storage.locking import RunLock, PlatformFileLock
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +26,7 @@ class LockingVersion(Enum):
 
 class LockType(Enum):
     SOFT_LOCK = 0
-    UNIX_LOCK = 1
+    PLATFORM_LOCK = 1
 
 
 @dataclass(frozen=True)
@@ -97,13 +97,13 @@ def get_run_lock_info(self, run_hash: str) -> LockInfo:
                 soft_lock_path = lock_dir / self.softlock_fname(run_hash)
                 if lock_path.exists():
                     try:
-                        lock = UnixFileLock(lock_path, timeout=0)
+                        lock = PlatformFileLock(lock_path, timeout=0)
                         with lock.acquire():
                             pass
                     except Timeout:
                         locked = True
                         lock_version = LockingVersion.LEGACY
-                        lock_type = LockType.UNIX_LOCK
+                        lock_type = LockType.PLATFORM_LOCK
                 elif soft_lock_path.exists():
                     locked = True
                     created_at = datetime.datetime.fromtimestamp(soft_lock_path.stat().st_mtime)
@@ -139,8 +139,8 @@ def release_locks(self, run_hash: str, force: bool) -> bool:
             for container_dir in ('meta', 'seqs'):
                 soft_lock_path = self.repo_path / container_dir / 'locks' / self.softlock_fname(run_hash)
                 soft_lock_path.unlink(missing_ok=True)
-                unix_lock_path = self.repo_path / container_dir / 'locks' / run_hash
-                unix_lock_path.unlink(missing_ok=True)
+                platform_lock_path = self.repo_path / container_dir / 'locks' / run_hash
+                platform_lock_path.unlink(missing_ok=True)
 
             # Force-release run lock
             lock_path.unlink(missing_ok=True)

aim/sdk/utils.py

@@ -20,7 +20,8 @@ def search_aim_repo(path) -> Tuple[Any, bool]:
         if os.path.exists(repo_path) and os.path.isdir(repo_path):
             found = True
             return path, found
-        if path == '/':
+        # count parts to support both Unix (/) and Windows roots (C:\)
+        if len(pathlib.Path(path).parts) == 1:
             return None, found
         path = os.path.dirname(path)
 
@@ -36,7 +37,7 @@ def clean_repo_path(repo_path: Union[str, pathlib.Path]) -> str:
     if not isinstance(repo_path, str) or not repo_path:
         return ''
 
-    repo_path = repo_path.strip().rstrip('/')
+    repo_path = repo_path.strip().rstrip('/').rstrip('\\')
 
     if isinstance(repo_path, pathlib.Path):
         repo_path = str(repo_path)

aim/storage/locking.py

@@ -1,7 +1,8 @@
 import os
 import logging
+import platform
 
-from filelock import BaseFileLock, SoftFileLock, UnixFileLock, has_fcntl
+from filelock import BaseFileLock, SoftFileLock, UnixFileLock, WindowsFileLock, has_fcntl
 
 from cachetools.func import ttl_cache
 from psutil import disk_partitions
@@ -10,6 +11,10 @@
 
 logger = logging.getLogger(__name__)
 
+if platform.system() == 'Windows':
+    PlatformFileLock = WindowsFileLock
+else:
+    PlatformFileLock = UnixFileLock
 
 class FileSystemInspector:
     """
@@ -142,7 +147,7 @@ def AutoFileLock(
         file lock.
     """
     if not FileSystemInspector.needs_soft_lock(lock_file):
-        return UnixFileLock(lock_file, timeout)
+        return PlatformFileLock(lock_file, timeout)
     else:
         # Cleaning lock files is not required by `FileLock`. The leftover lock files
         # (potentially from previous versions) could be interpreted as *acquired*
@@ -152,10 +157,10 @@ def AutoFileLock(
 
 
 class DualLock:
-    """ Custom lock that uses both UnixLock and SoftFileLock"""
+    """ Custom lock that uses both UnixFileLock/WindowsFileLock and SoftFileLock"""
     def __init__(self, lock_path: Union[str, os.PathLike], timeout: float = -1):
         self._lock_path = str(lock_path)
-        self._lock = UnixFileLock(self._lock_path, timeout)
+        self._lock = PlatformFileLock(self._lock_path, timeout)
 
         self._soft_lock_path = f'{self._lock_path}.softlock'
         self._soft_lock = SoftFileLock(self._soft_lock_path, timeout)

aim/storage/rockscontainer.pyx

@@ -618,5 +618,5 @@ class RocksContainerItemsIterator(ContainerItemsIterator):
 
 class LockableRocksContainer(RocksContainer):
     def get_lock_cls(self):
-        """Use both Unix file-locks and Soft file-locks to cover all the scenarios and avoid corruptions."""
+        """Use both Unix/Windows file-locks and Soft file-locks to cover all the scenarios and avoid corruptions."""
         return DualLock

docs/source/using/windows.md

@@ -0,0 +1,80 @@
+## Installing on Windows
+
+### Prerequisites
+
+Visual Studio 2019 or 2022. The free edition (Community Edition) is supported.
+
+### Overview
+
+Create an empty directory and place inside the build scripts (`.bat` files) with the content described below. Then run them in sequential order.
+
+### 1-build-deps.bat
+
+```batchfile
+git clone -b 2022.11.14 https://github.com/Microsoft/vcpkg.git
+
+cd vcpkg
+
+call bootstrap-vcpkg.bat -disableMetrics
+
+vcpkg install rocksdb[bzip2,lz4,snappy,zlib,zstd]:x64-windows-static-md
+
+cd ..
+```
+
+### 2-create-venv.bat
+
+You need to edit the script to select the Python version that you require. You can either use the Windows Python launcher (`py -3.10`) or you can directly specify the path to the Python installation (`C:\Program Files\Python 3.10\python.exe`, ...)
+
+```batchfile
+py -3.10 -m venv venv
+rem "C:\Program Files\Python 3.10\python.exe" -m venv venv
+
+call venv\Scripts\activate.bat
+
+python -m pip install -U pip setuptools wheel
+
+pip install pytest Cython==3.0.0.a9
+```
+
+### 3-build-aimrocks.bat
+
+```batchfile
+call venv\Scripts\activate.bat
+
+git clone https://github.com/aimhubio/aimrocks.git
+
+cd aimrocks
+
+python setup.py bdist_wheel
+
+pip install --find-links=dist aimrocks
+
+cd ..
+```
+
+### 4-build-aim.bat
+
+```batchfile
+call venv\Scripts\activate.bat
+
+git clone https://github.com/aimhubio/aim.git
+
+cd aim
+
+python setup.py bdist_wheel
+
+pip install --find-links=dist aim
+
+cd ..
+```
+
+### 5-run-aim.bat
+
+```batchfile
+call venv\Scripts\activate.bat
+
+aim version
+aim init
+aim up
+```

setup.py

@@ -132,16 +132,30 @@ def run(self):
 INCLUDE_DIRS = [lib_utils.get_include_dir()]
 LIB_DIRS = [lib_utils.get_lib_dir()]
 LIBS = lib_utils.get_libs()
-COMPILE_ARGS = [
-    '-std=c++11',
-    '-O3',
-    '-Wall',
-    '-Wextra',
-    '-Wconversion',
-    '-fno-strict-aliasing',
-    '-fno-rtti',
-    '-fPIC'
-]
+if platform.system() == 'Windows':
+    COMPILE_ARGS = [
+        '/std:c++latest',
+        '/permissive-',
+        '/W3',
+        '/O2',
+        '/EHsc',
+        '/GL'
+    ]
+    LINK_ARGS = [
+        '/LTCG'
+    ]
+else:
+    COMPILE_ARGS = [
+        '-std=c++11',
+        '-O3',
+        '-Wall',
+        '-Wextra',
+        '-Wconversion',
+        '-fno-strict-aliasing',
+        '-fno-rtti',
+        '-fPIC'
+    ]
+    LINK_ARGS = []
 CYTHON_SCRITPS = [
     ('aim.storage.hashing.c_hash', 'aim/storage/hashing/c_hash.pyx'),
     ('aim.storage.hashing.hashing', 'aim/storage/hashing/hashing.py'),
@@ -175,6 +189,7 @@ def configure_extension(name: str, path: str):
         libraries=LIBS,
         library_dirs=LIB_DIRS,
         extra_compile_args=COMPILE_ARGS,
+        extra_link_args=LINK_ARGS,
     )
 
 

tests/conftest.py

@@ -20,7 +20,7 @@ def _init_test_repo():
 
 
 def _cleanup_test_repo(path):
-    shutil.rmtree(TEST_REPO_PATH)
+    shutil.rmtree(path)
 
 
 def _upgrade_api_db():