various minor improvements

.github/actions/get-src/action.yaml

@@ -12,7 +12,7 @@ runs:
       run: |
         cd "$GITHUB_WORKSPACE" && \
           sudo apt-get update && sudo apt-get install -y ruby && \
-          (./build-scripts/install-deps.rb --os ${{ inputs.install_also }} || true) && \
+          (./build-scripts/install-deps.rb --os || true) && \
           ./build-scripts/install-deps.rb --os ${{ inputs.install_also }} && \
           git config --global user.email "[email protected]" && \
           git config --global user.name "CI worker" && \

.github/workflows/genvm-test-template.yaml

@@ -18,6 +18,9 @@ jobs:
     runs-on: ${{ inputs.runs_on }}
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
       - name: Get source
         uses: ./.github/actions/get-src
         with:
@@ -57,7 +60,7 @@ jobs:
         env:
           OPENAIKEY: ${{ secrets.OPENAIKEY }}
         run: |
-          OPENAIKEY="" python3 -m pip install -r genvm/testdata/runner/requirements.txt && \
+          OPENAIKEY="" python3.12 -m pip install -r genvm/testdata/runner/requirements.txt && \
           OPENAIKEY="" docker run --detach --rm --network host "$(docker build -q -t genvm/modules-webdriver -f genvm/modules/default-impl/web-funcs/webdriver.dockerfile genvm/modules/default-impl/web-funcs/)" && \
           sleep 5 && \
-          ./genvm/testdata/runner/run.py
+          python3.12 ./genvm/testdata/runner/run.py

.github/workflows/release.yml

@@ -17,26 +17,26 @@ jobs:
       contents: write  # Needed for creating releases
     steps:
       - uses: actions/checkout@v4
-      
+
       - name: Get source
         uses: ./.github/actions/get-src
         with:
-          install_also: --genvm --rust
+          install_also: --genvm --rust --wasi
 
       - name: Install tools and configure
         run: |
-          cd "$GITHUB_WORKSPACE"
-          source env.sh
+          cd "$GITHUB_WORKSPACE" && \
+          source env.sh && \
           ./tools/ya-build/ya-build config --preload .ci/release-conf.rb
 
       - name: Build package
         run: |
-          cd "$GITHUB_WORKSPACE"
-          source env.sh
-          ninja -v -C build tags/all
-          tree build/out
-          pushd build/out
-          zip -r -9 ../package.zip *
+          cd "$GITHUB_WORKSPACE" && \
+          source env.sh && \
+          ninja -v -C build tags/all && \
+          tree build/out && \
+          pushd build/out && \
+          zip -r -9 ../package.zip * && \
           popd
 
       - name: Create Release
@@ -47,4 +47,4 @@ jobs:
           name: Release ${{ github.ref_name }}
           draft: false
           prerelease: false
-          generate_release_notes: true
+          generate_release_notes: true

build-scripts/cargo.rb

@@ -4,7 +4,8 @@
 
 class CargoBuildTarget < Target
 	attr_reader :output_file
-	def initialize(dir, name, target, profile, features, flags)
+	def initialize(dir, name, target, profile, features, flags, env)
+		@env = env
 		@flags = flags
 		@features = features
 		@target_dir = CONFIGURATOR.root_build.join('generated', 'rust-target') # dir.join('target')
@@ -48,6 +49,13 @@ def initialize(dir, name, target, profile, features, flags)
 
 	protected def dump_rules_impl(buf)
 		buf << "  WD = #{Shellwords.escape @dir}\n"
+		if @env.size > 0
+			buf << "  ENV = env"
+			@env.each { |k, v|
+				buf << ' ' << k << '=' << Shellwords.escape(v).gsub(/\\=/, '=')
+			}
+			buf << "\n"
+		end
 		if @is_lib
 			buf << "  FLAGS = --lib"
 		else
@@ -93,14 +101,14 @@ def add_deps(*deps)
 
 add_rule(<<-EOF
 rule CARGO_BUILD
-  command = cd $WD && cargo build $FLAGS && touch $out
+  command = cd $WD && env $ENV cargo build $FLAGS && touch $out
   pool = console
   description = $DESC
 
 EOF
 )
 
-self.define_singleton_method(:target_cargo_build) do |out_file: nil, dir: nil, name:, target: nil, profile: "debug", features: [], flags: [], **kwargs, &blk|
+self.define_singleton_method(:target_cargo_build) do |out_file: nil, dir: nil, name:, target: nil, profile: "debug", features: [], flags: [], env: {}, **kwargs, &blk|
 	if target.nil?
 		@dflt_target ||= Proc.new {
 			o, e, s = Open3.capture3('rustc --version --verbose')
@@ -115,7 +123,7 @@ def add_deps(*deps)
 		dir = cur_src
 	end
 
-	trg = CargoBuildTarget.new(dir, name, target, profile, features, flags)
+	trg = CargoBuildTarget.new(dir, name, target, profile, features, flags, env)
 
 	if out_file.nil?
 		return return_target(trg, **kwargs, &blk)

build-scripts/install-deps.rb

@@ -11,6 +11,7 @@
 require 'rubygems/package'
 require 'zlib'
 require 'net/http'
+require 'shellwords'
 
 require 'optparse'
 
@@ -21,6 +22,7 @@
 OptionParser.new do |opts|
 	opts.on '--genvm'
 	opts.on '--rust'
+	opts.on '--rust-det'
 	opts.on '--os'
 	opts.on '--wasi'
 	opts.on '--test'
@@ -34,6 +36,49 @@
 	end
 	"#{severity.ljust(5)} #{msg}\n"
 end
+$logger = logger
+
+def run_command(*cmd, chdir: nil)
+	$logger.info("running #{cmd} at #{chdir || Dir.pwd}")
+	buf = String.new
+	kws = {}
+	if not chdir.nil?
+		kws[:chdir] = chdir
+	end
+	Open3.popen2e(*cmd.map { |s| s.to_s }, **kws) { |stdin, stdout, wait_thr|
+		stdin.close()
+		stdout.each_line { |l|
+			puts "\t#{l}"
+			buf << l << "\n"
+		}
+		exit_status = wait_thr.value
+		if exit_status != 0
+			raise "command #{cmd.map{ |x| Shellwords.escape x }.join(' ')} failed"
+		end
+	}
+	buf
+end
+
+def find_executable(name)
+	paths = ENV['PATH'].split(':')
+	paths << '/usr/bin'
+	paths << '/bin'
+	paths << "#{ENV['HOME']}/.local/bin"
+	paths << "#{ENV['HOME']}/.cargo/bin"
+	paths.each { |p|
+		check = ['', '.elf', '.exe']
+		check.each { |c|
+			cur_p = Pathname.new(p).join("#{name}#{c}")
+			if cur_p.exist?()
+				$logger.debug("located #{name} at #{cur_p}")
+				return cur_p
+			end
+		}
+	}
+	return nil
+end
+
+$bash = find_executable 'bash'
 
 TARGET_TRIPLE = Proc.new do
 	o, e, s = Open3.capture3('rustc --version --verbose')
@@ -84,19 +129,18 @@
 download_dir.mkpath()
 
 logger.debug("download dir is #{download_dir}")
-$logger = logger
 
 def load_packages_from_lists(dir)
 	$logger.info("downloading #{dir} packages")
 	case OS
 	when 'linux'
 		if Pathname.new('/etc/lsb-release').exist?()
-			puts `/usr/bin/bash "#{Pathname.new(__FILE__).parent.join('src', dir, 'ubuntu.sh')}"`
+			run_command $bash, Pathname.new(__FILE__).parent.join('src', dir, 'ubuntu.sh')
 		else
 			$logger.error("auto install of packages for linux excluding ubuntu is not supported")
 		end
 	when 'macos'
-		puts `sh "#{Pathname.new(__FILE__).parent.join('src', dir, 'brew.sh')}"`
+		run_command $bash, Pathname.new(__FILE__).parent.join('src', dir, 'brew.sh')
 	else
 		$logger.error("auto install of packages for your os is not supported")
 	end
@@ -110,40 +154,28 @@ def load_packages_from_lists(dir)
 	logger.error("ruby must be at least 3.0, yours is #{RUBY_VERSION}")
 end
 
-def find_executable(name)
-	paths = ENV['PATH'].split(':')
-	paths << '/usr/bin'
-	paths << '/bin'
-	paths << "#{ENV['HOME']}/.local/bin"
-	paths << "#{ENV['HOME']}/.cargo/bin"
-	paths.each { |p|
-		check = ['', '.elf', '.exe']
-		check.each { |c|
-			cur_p = Pathname.new(p).join("#{name}#{c}")
-			if cur_p.exist?()
-				return cur_p
-			end
-		}
-	}
-	return nil
-end
-
-if options[:rust]
+if options[:rust] || options[:'rust-det']
 	rustup = find_executable('rustup')
 	if rustup.nil?
 		logger.debug("downloading rust")
-		`curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --component rust-fmt`
-		rustup = ENV['HOME'] + "/.cargo/bin/rustup"
+		puts `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --component rust-fmt`
+		rustup = find_executable 'rustup'
 	else
 		logger.debug("rustup is already installed at #{rustup}")
 	end
-	`cd "#{root}" && #{rustup} show active-toolchain || #{rustup} toolchain install`
+	raise "rustup not found" if rustup.nil?
+
+	puts `cd "#{root}" && #{rustup} show active-toolchain || #{rustup} toolchain install`
+	run_command(rustup, 'component', 'add', 'rustfmt', chdir: root)
+end
 
-	cur_toolchain = `#{rustup} show active-toolchain`
-	cur_toolchain = cur_toolchain.strip
+if options[:'rust-det']
+	ext_path = root.join('runners', 'cpython-and-ext', 'extension')
+	cur_toolchain = run_command(rustup, 'show', 'active-toolchain', chdir: ext_path)
+	cur_toolchain = cur_toolchain.lines.map { |l| l.strip }.filter { |l| l.size != 0 }.last
 	cur_toolchain = /^([a-zA-Z0-9\-_]+)/.match(cur_toolchain)[1]
 	logger.debug("installing for toolchain #{cur_toolchain}")
-	`cd "#{root}" && #{rustup} target add --toolchain #{cur_toolchain} wasm32-wasip1`
+	run_command(rustup, 'target', 'add', '--toolchain', cur_toolchain, 'wasm32-wasip1', chdir: ext_path)
 end
 
 if options[:wasi]

build-scripts/publish_runner.rb

@@ -32,7 +32,7 @@
 	target_command(
 		output_file: out_file,
 		commands: [
-			[config.python, publish_script, runner_publish_config_file]
+			[config.tools.python3, publish_script, runner_publish_config_file]
 		],
 		dependencies: [publish_script, runner_json] + dependencies + fdeps
 	) {

build-scripts/src/os/ubuntu.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 set -ex
-sudo apt-get install -y pkg-config ninja-build curl git python3 zip unzip tar tree
+sudo apt-get install -y pkg-config ninja-build curl git python3 zip unzip tar tree mold
 sudo apt-get satisfy -y 'ruby (>= 3.0)'
 git lfs install

env.sh

@@ -2,6 +2,8 @@
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
+export ASAN_OPTIONS=detect_leaks=1
+
 export PATH="$SCRIPT_DIR/tools/git-third-party:$SCRIPT_DIR/tools/ya-build:$PATH"
 if [ -f "$SCRIPT_DIR/.env" ]
 then