diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6fbb86b019..214262b181 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -85,6 +85,7 @@ jobs: - env: LEIN_PROFILES: ${{ matrix.lein-profile }} PDB_QUERY_OPTIMIZE_DROP_UNUSED_JOINS: ${{ matrix.drop-joins }} + NO_ACCEPTANCE: true run: ci/bin/prep-and-run-in github ${{ matrix.flavor }} - uses: actions/upload-artifact@v3 with: diff --git a/Gemfile b/Gemfile index eae18f16f0..f6da3b5ef1 100644 --- a/Gemfile +++ b/Gemfile @@ -65,10 +65,10 @@ if ENV['NO_ACCEPTANCE'] != 'true' # use the pinned version gem 'beaker', '~> 4.1' end + gem 'beaker-hostgenerator', '~> 2.2.3' + gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2') + gem 'beaker-vmpooler', *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3") + gem 'beaker-puppet', '~> 1.0' + gem 'faraday', '~> 1.8.0' end - gem 'beaker-hostgenerator', '~> 1.12' - gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2') - gem 'beaker-vmpooler', *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3") - gem 'beaker-puppet', '~> 1.0' - gem 'faraday', '~> 1.8.0' end diff --git a/ext/bin/config-puppet-test-ref b/ext/bin/config-puppet-test-ref index cf314abda6..51d56ee894 100755 --- a/ext/bin/config-puppet-test-ref +++ b/ext/bin/config-puppet-test-ref @@ -49,7 +49,9 @@ case "$#" in esac # Install and update Ruby dependencies from top-level Gemfile -bundle install --without acceptance --path vendor/bundle --retry=10 +bundle config set --local path vendor/bundle +bundle config set --local without acceptance +bundle install --retry=10 bundle update # Print out info on Puppet dependency bundle info puppet || true # Not all versions appear to have info @@ -59,6 +61,6 @@ bundle info puppet || true # Not all versions appear to have info cd vendor # This will something like .../vendor/bundle/ruby/2.7.0/bundler/gems/puppet-ae5379e03311 # It is a Puppet git repository installed by bundler -puppet_path="$(bundle show puppet)" +puppet_path="$(bundle info puppet --path)" # Create a symlink at `vendor/puppet` that points to Puppet repository "$top/ext/bin/symlink-relative-to" "$puppet_path" puppet diff --git a/ext/bin/config-puppetserver-test-ref b/ext/bin/config-puppetserver-test-ref index e86064881b..dd41b491d5 100755 --- a/ext/bin/config-puppetserver-test-ref +++ b/ext/bin/config-puppetserver-test-ref @@ -63,7 +63,7 @@ if test -d puppetserver; then git -C puppetserver clean -fdx else # If repo doesn't exist, clone from GitHub - git clone --depth 10 -b "$pupsrv_ref" https://github.com/puppetlabs/puppetserver + git clone -b "$pupsrv_ref" https://github.com/puppetlabs/puppetserver fi (export PUPPETSERVER_HEAP_SIZE=1G @@ -72,6 +72,7 @@ fi # If we're not testing against the git tree (say in jenkins), don't install. if test -z "$PDB_NO_PUPPETSERVER_INSTALL"; then lein install + lein uberjar fi # Cache Puppet Server version in a file in ext/test-conf directory dep_ver="$(lein-pprint :version)" diff --git a/test/puppetlabs/puppetdb/integration/fixtures.clj b/test/puppetlabs/puppetdb/integration/fixtures.clj index 3eb449f848..c4affe51b4 100644 --- a/test/puppetlabs/puppetdb/integration/fixtures.clj +++ b/test/puppetlabs/puppetdb/integration/fixtures.clj @@ -13,13 +13,14 @@ [puppetlabs.puppetdb.testutils.db :as dbutils] [puppetlabs.puppetdb.testutils.services :as svc-utils] [puppetlabs.trapperkeeper.app :as tk-app] - [puppetlabs.trapperkeeper.bootstrap :as tk-bootstrap] [puppetlabs.trapperkeeper.config :as tk-config] [puppetlabs.trapperkeeper.testutils.bootstrap :as tkbs] [yaml.core :as yaml] [puppetlabs.puppetdb.time :as time] [puppetlabs.puppetdb.utils :as utils]) - (:import [com.typesafe.config ConfigValueFactory])) + (:import + (com.typesafe.config ConfigValueFactory) + (java.lang ProcessBuilder ProcessBuilder$Redirect))) (defprotocol TestServer (server-info [this])) @@ -154,13 +155,13 @@ ;;; Puppet Server fixture -(defrecord PuppetServerTestServer [info-map files-to-cleanup app] +(defrecord PuppetServerTestServer [info-map files-to-cleanup process] TestServer (server-info [_] info-map) java.lang.AutoCloseable (close [_] - (tk-app/stop app) + (doto process .destroy .waitFor) ;; Assumes destroy sends a SIGTERM (doseq [f files-to-cleanup] (fs/delete f)))) (def dev-config-file "./test-resources/puppetserver/puppetserver.conf") @@ -201,6 +202,21 @@ (clojure.string/join ","))}} (or overrides {}))))) +(defn- wait-for-server + [port timeout-ms] + (loop [n (/ timeout-ms 100)] + (when (neg? n) + (throw (ex-info (str "server not ready within " timeout-ms "ms") {}))) + (let [res (try + (svc-utils/get-ssl (str "https://localhost:" port "/status/v1/services")) + (catch java.net.ConnectException _ + ::nope))] + (cond + (= res ::nope) (do (Thread/sleep 100) (recur (dec n))) + (= 200 (:status res)) true + :else (throw (ex-info "Unexpected result from status endpoint while waiting" + res)))))) + (defn run-puppet-server-as [node-name pdb-servers config-overrides] (let [puppetdb-conf (io/file "target/puppetserver/master-conf/puppetdb.conf") puppet-conf (io/file "target/puppetserver/master-conf/puppet.conf") @@ -224,21 +240,37 @@ (write-puppetdb-terminus-config pdb-servers puppetdb-conf terminus-config-overrides) - (let [services (tk-bootstrap/parse-bootstrap-config! dev-bootstrap-file) - tmp-conf (ks/temp-file "puppetserver" ".conf") - _ (fs/copy dev-config-file tmp-conf) - port (svc-utils/open-port-num) - config (-> (tk-config/load-config (.getPath tmp-conf)) + (let [port (svc-utils/open-port-num) + config (-> (tk-config/load-config dev-config-file) (merge puppetserver-config-overrides) - (assoc-in [:webserver :ssl-port] port))] + (assoc-in [:webserver :ssl-port] port)) + config-file (ks/temp-file "puppetserver-conf" ".edn") + cmd ["java" "-cp" "puppetserver/target/puppet-server-release.jar" + "clojure.main" "-m" "puppetlabs.trapperkeeper.main" + "services" + "--bootstrap-config" dev-bootstrap-file + "-c" (.getPath config-file)] + adjust-env #(doto ^java.util.Map (.environment %) + (.remove "CLASSPATH")) + pb (doto (ProcessBuilder. cmd) + (.redirectOutput ProcessBuilder$Redirect/INHERIT) + (.redirectError ProcessBuilder$Redirect/INHERIT) + adjust-env) + _ (spit config-file (pr-str config)) + process (.start pb)] + (.addShutdownHook (Runtime/getRuntime) + (doto (Thread. #(.destroy process)) + (.setName (str "Subprocess cleanup for " process)))) + (log/info (str "Started puppetserver on port " port)) + (wait-for-server port tu/default-timeout-ms) (PuppetServerTestServer. {:hostname "localhost" :port port :code-dir "target/puppetserver/master-code" :conf-dir "target/puppetserver/master-conf"} - [(.getPath tmp-conf) + [(.getPath config-file) "target/puppetserver/master-conf" "target/puppetserver/master-code"] - (tkbs/bootstrap-services-with-config services config))))) + process)))) (defn run-puppet-server [pdb-servers config-overrides] (run-puppet-server-as "localhost" pdb-servers config-overrides))