From 377a042f801bc299903e5fc68510fe5294b0fad7 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Mon, 26 Sep 2022 13:38:44 -0300 Subject: [PATCH 001/260] fix: use Orocos::ProcessBase instead of name service We are changing orocos to get the task's IOR through a pipe with orogen instead of the name service to discover the task. See https://github.com/rock-core/tools-orogen/pull/7 and https://github.com/rock-core/tools-orocosrb/pull/149 --- lib/syskit/roby_app/remote_processes/process.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/syskit/roby_app/remote_processes/process.rb b/lib/syskit/roby_app/remote_processes/process.rb index fbeb9b050..9a1d7f152 100644 --- a/lib/syskit/roby_app/remote_processes/process.rb +++ b/lib/syskit/roby_app/remote_processes/process.rb @@ -42,12 +42,6 @@ def dead! @alive = false end - # Returns the task context object for the process' task that has this - # name - def task(task_name) - process_client.name_service.get(task_name, process: self) - end - # Cleanly stop the process # # @see kill! From a444658557811ded8cb01c2f50ed186fa2888e3d Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 28 Sep 2022 10:31:06 -0300 Subject: [PATCH 002/260] feat: resolve the deployments in a single non-threaded call to the process server --- lib/syskit/deployment.rb | 71 +++++++++---------- .../runtime/update_deployment_states.rb | 40 +++++++++++ 2 files changed, 75 insertions(+), 36 deletions(-) diff --git a/lib/syskit/deployment.rb b/lib/syskit/deployment.rb index 15ddeead3..c3d2a2606 100644 --- a/lib/syskit/deployment.rb +++ b/lib/syskit/deployment.rb @@ -468,16 +468,6 @@ def log_port?(port) end end - on :start do |_event| - handles_from_plan = {} - each_parent_object(Roby::TaskStructure::ExecutionAgent) do |task| - if orocos_task = task.orocos_task - handles_from_plan[task.orocos_name] = orocos_task - end - end - schedule_ready_event_monitor(handles_from_plan) - end - # @api private # # Event used to quit the ready monitor started by @@ -487,49 +477,58 @@ def log_port?(port) attr_reader :quit_ready_event_monitor # @api private - # - # Schedule a promise to resolve the task handles - # - # It will reschedule itself until the process is ready, and will - # emit the ready event when it happens - def schedule_ready_event_monitor( - handles_from_plan, ready_polling_period: self.ready_polling_period - ) - distance_to_syskit = self.distance_to_syskit - promise = execution_engine.promise(description: "#{self}:ready_event_monitor") do - resolve_remote_task_handles(handles_from_plan) + # This schedules an asynchronous process to connect to the tasks before + # ready is emitted. I.e. the process is not ready after the method returns. + # + # @param [{String=>String}] ior_mappings the mappings of the form + # { task_name => ior }. This is necessary because the Syskit remote process has + # no information about the IORs until now. + def update_remote_tasks(ior_mappings) + orocos_process.define_ior_mappings(ior_mappings) + begin + remote_tasks = orocos_process.resolve_all_tasks + rescue Orocos::IORNotRegisteredError, ArgumentError => e + ready_event.emit_failed(e) + return + end + + promise = execution_engine.promise( + description: "#{self}#update_remote_tasks#resolve handles" + ) do + resolve_remote_task_handles(remote_tasks) end - promise.on_success(description: "#{self}#schedule_ready_event_monitor#emit") do |remote_tasks| - if running? && !finishing? && remote_tasks - @remote_task_handles = remote_tasks + promise.on_success( + description: "#{self}#update_remote_tasks#success" + ) do |remote_task_handles| + if running? && !finishing? + @remote_task_handles = remote_task_handles ready_event.emit end end promise.on_error(description: "#{self}#emit_failed") do |reason| - ready_event.emit_failed(reason) if !finishing? || !finished? + ready_event.emit_failed(reason) unless finishing? || finished? end + ready_event.pending([]) ready_event.achieve_asynchronously( promise, emit_on_success: false, on_failure: :nothing ) end - def resolve_remote_task_handles( - handles_from_plan, ready_polling_period: self.ready_polling_period - ) - until (handles = orocos_process.resolve_all_tasks(handles_from_plan)) - return if quit_ready_event_monitor.set? - - sleep ready_polling_period - end + def resolve_remote_task_handles(remote_tasks) + return if quit_ready_event_monitor.set? - handles.transform_values do |remote_task| - state_reader, state_getter = create_state_access(remote_task, distance: distance_to_syskit) + remote_tasks.transform_values do |remote_task| + state_reader, state_getter = + create_state_access(remote_task, distance: distance_to_syskit) properties = remote_task.property_names.map do |p_name| p = remote_task.raw_property(p_name) [p, p.raw_read.freeze] end current_configuration = CurrentTaskConfiguration.new(nil, [], Set.new) - RemoteTaskHandles.new(remote_task, state_reader, state_getter, properties, false, current_configuration) + RemoteTaskHandles.new( + remote_task, state_reader, state_getter, + properties, false, current_configuration + ) end end diff --git a/lib/syskit/runtime/update_deployment_states.rb b/lib/syskit/runtime/update_deployment_states.rb index 6bd8236b7..0187130b9 100644 --- a/lib/syskit/runtime/update_deployment_states.rb +++ b/lib/syskit/runtime/update_deployment_states.rb @@ -10,6 +10,11 @@ def self.update_deployment_states(plan) # #cleanup_dead_connections, thus avoiding to disconnect connections # between already-dead processes + handle_dead_deployments(plan) + trigger_ready_deployments(plan) + end + + def self.handle_dead_deployments(plan) all_dead_deployments = Set.new server_config = Syskit.conf.each_process_server_config.to_a server_config.each do |config| @@ -33,6 +38,27 @@ def self.update_deployment_states(plan) end end + def self.trigger_ready_deployments(plan) + not_ready_deployments = find_all_not_ready_deployments(plan) + not_ready_deployments.each do |process_server_name, deployments| + server_config = Syskit.conf.process_server_config_for(process_server_name) + wait_result = server_config.client.wait_running( + *deployments.map { |d| d.arguments[:process_name] } + ) + wait_result.each do |process_name, result| + next unless result + + deployment = deployments.find { |d| d.process_name == process_name } + + if result[:error] + deployment.ready_event.emit_failed(result[:error]) + elsif result[:iors] + deployment.update_remote_tasks(result[:iors]) + end + end + end + end + def self.abort_process_server(plan, process_server) client = process_server.client # Before we can terminate Syskit, we need to abort all @@ -42,5 +68,19 @@ def self.abort_process_server(plan, process_server) deployments.each { |t| t.aborted_event.emit if !t.pending? && !t.finished? } Syskit.conf.remove_process_server(process_server.name) end + + def self.find_all_not_ready_deployments(plan) + # Must also check if the deployment task is finishing in case it is + # stopped before becoming ready and if the ready event is pending, + # which would mean that it the deployment is already updating its + # remote tasks + valid_running_deployment_tasks = + plan.find_tasks(Syskit::Deployment) + .running + .find_all do |dep_task| + !dep_task.ready? && !dep_task.finishing? && !dep_task.ready_event.pending? + end + valid_running_deployment_tasks.group_by { |t| t.arguments[:on] } + end end end From c87d42432d9a01381450874c878cc3167187a0bd Mon Sep 17 00:00:00 2001 From: Jhonas Date: Thu, 20 Oct 2022 10:30:57 -0300 Subject: [PATCH 003/260] feat: add wait running command to remote process client and server This command is used to wait for the process to be running, which means that the process has received the IOR from the orogen tasks. The IOR is then mapped to each task, and used to create the TaskContexts. The server returns a Hash reporting any exceptions the occured, and if there wasnt one, it reports the iors for the process' tasks or nil if no iors were received yet. --- .../roby_app/remote_processes/client.rb | 8 ++++++ .../roby_app/remote_processes/protocol.rb | 1 + .../roby_app/remote_processes/server.rb | 28 +++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/lib/syskit/roby_app/remote_processes/client.rb b/lib/syskit/roby_app/remote_processes/client.rb index 758194c47..d99db4952 100644 --- a/lib/syskit/roby_app/remote_processes/client.rb +++ b/lib/syskit/roby_app/remote_processes/client.rb @@ -314,6 +314,14 @@ def kill_all(cleanup: false, hard: true) Marshal.load(socket) end + def wait_running(*process_names) + socket.write(COMMAND_WAIT_RUNNING) + Marshal.dump(process_names, socket) + wait_for_answer do + return Marshal.load(socket) + end + end + def join(deployment_name) process = processes[deployment_name] return unless process diff --git a/lib/syskit/roby_app/remote_processes/protocol.rb b/lib/syskit/roby_app/remote_processes/protocol.rb index 49a0f52f2..8e208ea47 100644 --- a/lib/syskit/roby_app/remote_processes/protocol.rb +++ b/lib/syskit/roby_app/remote_processes/protocol.rb @@ -12,6 +12,7 @@ module RemoteProcesses COMMAND_END = "E" COMMAND_QUIT = "Q" COMMAND_KILL_ALL = "K" + COMMAND_WAIT_RUNNING = "W" COMMAND_LOG_UPLOAD_FILE = "U" COMMAND_LOG_UPLOAD_STATE = "X" diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 1cff547dd..9779d6cf1 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -419,6 +419,34 @@ def handle_command(socket) # :nodoc: state = log_upload_state socket.write RET_YES socket.write Marshal.dump(state) + elsif cmd_code == COMMAND_WAIT_RUNNING + result = {} + process_names = Marshal.load(socket) + process_names.each do |p_name| + if (p = @processes[p_name]) + begin + iors = p.wait_running(0) + result[p_name] = ({ iors: iors } if iors) + rescue Orocos::NotFound => e + Server.warn(e.message) + result[p_name] = { error: e.message } + rescue Orocos::InvalidIORMessage => e + Server.warn(e.message) + result[p_name] = { error: e.message } + end + else + Server.warn("no process named #{p_name} to wait running") + result[p_name] = { + error: "no process named #{p_name} to wait running" + } + end + rescue RuntimeError => e + process_names.each do |process_name| + result[process_name] ||= { error: e.message } + end + end + socket.write(RET_YES) + Marshal.dump(result, socket) end true From f4b097c0cb281157f28bd0f93c603c0dfdfedd65 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Thu, 20 Oct 2022 10:27:07 -0300 Subject: [PATCH 004/260] feat: implement wait running interface for remote process --- lib/syskit/roby_app/remote_processes/process.rb | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/syskit/roby_app/remote_processes/process.rb b/lib/syskit/roby_app/remote_processes/process.rb index 9a1d7f152..0f44431af 100644 --- a/lib/syskit/roby_app/remote_processes/process.rb +++ b/lib/syskit/roby_app/remote_processes/process.rb @@ -29,6 +29,7 @@ def initialize(name, deployment_model, process_client, pid) @process_client = process_client @pid = pid @alive = true + @ior_mappings = {} super(name, deployment_model) end @@ -64,20 +65,12 @@ def running? @alive end - # Resolve all tasks within the deployment - # - # A deployment is usually considered ready when all its tasks can be - # resolved successfully - def resolve_all_tasks(cache = {}) - Orocos::Process.resolve_all_tasks(self, cache) do |task_name| - task(task_name) - end + def resolve_all_tasks + Orocos::Process.resolve_all_tasks(self) end - # Waits for the deployment to be ready. +timeout+ is the number of - # milliseconds we should wait. If it is nil, will wait indefinitely - def wait_running(timeout = nil) - Orocos::Process.wait_running(self, timeout) + def define_ior_mappings(mappings) + @ior_mappings = mappings end end end From f0854aad4385123306de0db17c23dd228cd1e6ca Mon Sep 17 00:00:00 2001 From: Jhonas Date: Thu, 20 Oct 2022 10:39:07 -0300 Subject: [PATCH 005/260] feat: implement wait running interface for unmanaged processes The unmanaged processes will still resolve their tasks using the name service (as they arent managed by syskit...). Wait running will wait on a promise that will try to resolve each task using the name service, then after they are resolved, the ior mappings will be filled using the tasks information to keep the same interface as the remote processes and ruby tasks. --- lib/syskit/roby_app/unmanaged_process.rb | 81 +++++++++++++------ .../roby_app/unmanaged_tasks_manager.rb | 18 +++++ 2 files changed, 75 insertions(+), 24 deletions(-) diff --git a/lib/syskit/roby_app/unmanaged_process.rb b/lib/syskit/roby_app/unmanaged_process.rb index 04968584c..916bf2676 100644 --- a/lib/syskit/roby_app/unmanaged_process.rb +++ b/lib/syskit/roby_app/unmanaged_process.rb @@ -82,11 +82,13 @@ def on_localhost? # process is expected to be running def initialize(process_manager, name, model, host_id: "unmanaged_process") @process_manager = process_manager - @deployed_tasks = nil + @deployed_tasks = {} @name_service = process_manager.name_service @host_id = host_id @quitting = Concurrent::Event.new super(name, model) + + @default_logger = false end # "Starts" this process @@ -97,7 +99,29 @@ def initialize(process_manager, name, model, host_id: "unmanaged_process") def spawn(_options = {}) @spawn_start = Time.now @last_warning = Time.now - @deployed_tasks = nil + @deployed_tasks = {} + + @iors_future = Concurrent::Promises.future do + name_service_get_all_tasks + end + end + + # Calls the name service until all of the tasks are resolved. Ignores whenever + # a Orocos::NotFound exception is raised. + # + # @raises RuntimeError + # @raises Orocos::CORBA::ComError + # @return [Hash] + def name_service_get_all_tasks + result = {} + until task_names.size == result.size + task_names.each do |name| + result[name] = name_service.get(name) + rescue Orocos::NotFound + next + end + end + result end # Verifies that the monitor thread is alive and well, or that the @@ -110,21 +134,17 @@ def verify_threads_state monitor_thread.join if monitor_thread && !monitor_thread.alive? end - def resolve_all_tasks(cache = {}) - resolved = model.task_activities.map do |t| - [t.name, (cache[t.name] ||= name_service.get(t.name))] - end - @deployed_tasks = Hash[resolved] - @monitor_thread = Thread.new do - monitor - end + # Returns the deployed tasks.The deployed tasks are resolved on wait_running, + # which is called by `update_deployment_states.rb` when making the deployment + # ready. + # + # @returns [Hash] + def resolve_all_tasks @deployed_tasks - rescue Orocos::NotFound, Orocos::ComError => e - if Time.now - @last_warning > 5 - Syskit.warn "waiting for unmanaged task: #{e}" - @last_warning = Time.now - end - nil + end + + def define_ior_mappings(ior_mappings) + @ior_mappings = ior_mappings end # Returns the component object for the given name @@ -133,12 +153,24 @@ def resolve_all_tasks(cache = {}) # @raise [ArgumentError] if the name is not the name of a task on # self def task(task_name) - raise "process not running yet" unless deployed_tasks - unless (task = deployed_tasks[task_name]) - raise ArgumentError, "#{task_name} is not a task of #{self}" - end + raise "process not running yet" unless ready? + + @deployed_tasks.fetch(task_name) + end + + # Waits until all the tasks are resolved or the timeout is due, registering + # the IORs of the resolved tasks and starting the monitor. + # + # @raises RuntimeError + # @raises Orocos::CORBA::ComError + # @return [Hash] the ior mappings + def wait_running(timeout = nil) + return unless (tasks = @iors_future.value!(timeout)) - task + @ior_mappings = tasks.transform_values(&:ior) + @deployed_tasks = tasks + @monitor_thread = Thread.new { monitor(tasks) } + @ior_mappings end # @api private @@ -168,10 +200,11 @@ def kill(_wait = false, **) # Implementation of the monitor thread, i.e. the thread that will # detect if the deployment disappears # + # @param [Hash] tasks map of task name to task # @param [Float] period polling period in seconds - def monitor(period: 0.1) + def monitor(tasks, period: 0.1) until quitting? - deployed_tasks.each_value do |task| + tasks.each_value do |task| task.ping rescue Orocos::ComError return # rubocop:disable Lint/NonLocalExitFromIterator @@ -195,7 +228,7 @@ def dead? # Returns true if the tasks have been successfully discovered def ready? - deployed_tasks + @iors_future.resolved? end # True if the process is running. This is an alias for running? diff --git a/lib/syskit/roby_app/unmanaged_tasks_manager.rb b/lib/syskit/roby_app/unmanaged_tasks_manager.rb index 73e7767ce..0b85c37eb 100644 --- a/lib/syskit/roby_app/unmanaged_tasks_manager.rb +++ b/lib/syskit/roby_app/unmanaged_tasks_manager.rb @@ -152,6 +152,24 @@ def wait_termination(_timeout = nil) dead_processes end + def wait_running(*process_names) + result = {} + process_names.each do |name| + if (p = processes[name]) + begin + ior_resolution = p.wait_running(0) + result[name] = { iors: ior_resolution } + rescue Orocos::NotFound, Orocos::CORBA::ComError => e + result[name] = { error: e.message } + end + else + result[name] = + { error: "#{name} was not found of the processes list" } + end + end + result + end + # Requests to stop the given deployment # # The call does not block until the process has quit. You will have to From fa437c8e515776f50806b7a51184a498fc1cbc17 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Fri, 30 Sep 2022 13:20:54 -0300 Subject: [PATCH 006/260] fix(test): add wait running interface for the deployment process fixtures --- test/test_deployment.rb | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/test/test_deployment.rb b/test/test_deployment.rb index 83a7d6c02..8d2fb8da0 100644 --- a/test/test_deployment.rb +++ b/test/test_deployment.rb @@ -23,14 +23,37 @@ def start(name, *) tasks.fetch(name) end + def wait_running(*process_names) + resolved = {} + process_names.each do |p_name| + resolved[p_name] = {} + @processes.each_value do |p| + resolved[p_name] = { iors: p.ior_mappings } + end + end + resolved + end + def disconnect; end end class ProcessFixture + + class ModelFixture + def extended_state_support? + false + end + end + attr_reader :process_server attr_reader :name_mappings + attr_reader :ior_mappings + attr_reader :model + attr_reader :property_names def initialize(process_server) @process_server = process_server @name_mappings = Hash["task" => "mapped_task_name"] + @ior_mappings = { "mapped_task_name" => "IOR" } + @property_names = [] end def get_mapped_name(name) @@ -41,7 +64,14 @@ def kill(*, **) process_server.killed_processes << self end - def resolve_all_tasks(*); end + def define_ior_mappings(ior_mappings) + @ior_mappings = ior_mappings + end + + def resolve_all_tasks + process_server.tasks + end + end describe Deployment do @@ -327,6 +357,7 @@ def mock_raw_port(task, port_name) @orocos_task = Orocos.allow_blocking_calls do Orocos::RubyTasks::TaskContext.new "test" end + process.define_ior_mappings({"mapped_task_name" => "IOR"}) end after do orocos_task.dispose @@ -450,7 +481,6 @@ def initialize_remote_handles(handles); end task = add_deployed_task task.orocos_task = orocos_task process.should_receive(:resolve_all_tasks).once - .with("mapped_task_name" => orocos_task) .and_return("mapped_task_name" => orocos_task) make_deployment_ready From 7e1579b29878ef21fc69686397845190fd70012c Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 5 Oct 2022 12:42:16 -0300 Subject: [PATCH 007/260] fix(test): update remote processes tests with wait running interface --- .../remote_processes/test_remote_processes.rb | 121 +++++++++++++++++- 1 file changed, 114 insertions(+), 7 deletions(-) diff --git a/test/roby_app/remote_processes/test_remote_processes.rb b/test/roby_app/remote_processes/test_remote_processes.rb index 1c0287a7b..8fcbacd36 100644 --- a/test/roby_app/remote_processes/test_remote_processes.rb +++ b/test/roby_app/remote_processes/test_remote_processes.rb @@ -97,19 +97,18 @@ start_and_connect_to_server end - it "can start a process on the server synchronously" do + it "can start a process on the server" do process = client.start( "syskit_tests_empty", "syskit_tests_empty", { "syskit_tests_empty" => "syskit_tests_empty" }, - wait: true, oro_logfile: nil, output: "/dev/null" + oro_logfile: nil, output: "/dev/null" ) assert process.alive? - assert(Orocos.allow_blocking_calls { Orocos.get("syskit_tests_empty") }) end it "raises if the deployment does not exist on the remote server" do assert_raises(OroGen::DeploymentModelNotFound) do - client.start "bla", "bla", Hash["sink" => "test"], wait: true + client.start "bla", "bla", Hash["sink" => "test"] end end @@ -128,12 +127,120 @@ root_loader.register_deployment_model(deployment) process = client.start( "syskit_tests_empty", "syskit_tests_empty", {}, - wait: true, oro_logfile: nil, output: "/dev/null" + oro_logfile: nil, output: "/dev/null" ) assert_same deployment, process.model end end + describe "waits for the process to be running" do + before do + start_and_connect_to_server + end + + it "returns a hash with information about a process and its tasks" do + client.start( + "syskit_tests_empty", "syskit_tests_empty", + { "syskit_tests_empty" => "syskit_tests_empty" }, + oro_logfile: nil, output: "/dev/null" + ) + result = nil + loop do + result = client.wait_running("syskit_tests_empty") + break if result["syskit_tests_empty"]&.key?(:iors) + end + + assert_match( + /^IOR/, + result["syskit_tests_empty"][:iors]["syskit_tests_empty"] + ) + end + + it "returns a hash without any info when the process didnt get its tasks ior" do + client.start( + "syskit_tests_empty", "syskit_tests_empty", + { "syskit_tests_empty" => "syskit_tests_empty" }, + oro_logfile: nil, output: "/dev/null" + ) + result = client.wait_running("syskit_tests_empty") + assert_equal({ "syskit_tests_empty" => nil }, result) + end + + it "reports a Orocos::NotFound error specific to a process" do + not_found_error_message = "syskit_tests_empty was started but crashed" + flexmock(Orocos::ProcessBase) + .new_instances + .should_receive(:wait_running) + .and_raise(Orocos::NotFound, not_found_error_message) + + client.start( + "syskit_tests_empty", "syskit_tests_empty", + { "syskit_tests_empty" => "syskit_tests_empty" }, + oro_logfile: nil, output: "/dev/null" + ) + result = client.wait_running("syskit_tests_empty") + expected = { + "syskit_tests_empty" => { error: not_found_error_message } + } + assert_equal(expected, result) + end + + it "reports a invalid ior message error specific to a process" do + ior_invalid_error_message = + "the ior message doesnt contain information about the following tasks:" \ + " [\"syskit_tests_empty_Logger\"]" + flexmock(Orocos::ProcessBase) + .new_instances + .should_receive(:wait_running) + .and_raise(Orocos::InvalidIORMessage, ior_invalid_error_message) + + client.start( + "syskit_tests_empty", "syskit_tests_empty", + { "syskit_tests_empty" => "syskit_tests_empty" }, + oro_logfile: nil, output: "/dev/null" + ) + result = client.wait_running("syskit_tests_empty") + expected = { + "syskit_tests_empty" => { error: ior_invalid_error_message } + } + assert_equal(expected, result) + end + + it "reports when the process name is not present in the processes' list" do + not_present_error = + "no process named another_syskit_tests_empty to wait running" + client.start( + "syskit_tests_empty", "syskit_tests_empty", + { "syskit_tests_empty" => "syskit_tests_empty" }, + oro_logfile: nil, output: "/dev/null" + ) + result = client.wait_running("another_syskit_tests_empty") + expected = { + "another_syskit_tests_empty" => { error: not_present_error } + } + assert_equal(expected, result) + end + + it "reports when a runtime error occured" do + runtime_error_message = "some runtime error occured" + flexmock(Orocos::ProcessBase) + .new_instances + .should_receive(:wait_running) + .and_raise(RuntimeError, runtime_error_message) + + client.start( + "syskit_tests_empty", "syskit_tests_empty", + { "syskit_tests_empty" => "syskit_tests_empty" }, + oro_logfile: nil, output: "/dev/null" + ) + result = client.wait_running("syskit_tests_empty") + expected = { + "syskit_tests_empty" => { error: runtime_error_message } + } + assert_equal(expected, result) + end + end + describe "stopping a remote process" do attr_reader :process before do @@ -141,7 +248,7 @@ @process = client.start( "syskit_tests_empty", "syskit_tests_empty", { "syskit_tests_empty" => "syskit_tests_empty" }, - wait: true, oro_logfile: nil, output: "/dev/null" + oro_logfile: nil, output: "/dev/null" ) end @@ -170,7 +277,7 @@ "syskit_tests_empty_#{i}", "syskit_tests_empty", { "syskit_tests_empty" => "syskit_tests_empty_#{i}", "syskit_tests_empty_Logger" => "syskit_tests_empty_#{i}_Logger" }, - wait: false, oro_logfile: nil, output: "/dev/null" + oro_logfile: nil, output: "/dev/null" ) end end From 708d596db0ec275847ac827c4877cd583414c1e9 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Thu, 20 Oct 2022 11:10:33 -0300 Subject: [PATCH 008/260] fix(test): update unit tests for deployment after trigger ready interface --- test/test_deployment.rb | 111 +++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 53 deletions(-) diff --git a/test/test_deployment.rb b/test/test_deployment.rb index 8d2fb8da0..f33c32c18 100644 --- a/test/test_deployment.rb +++ b/test/test_deployment.rb @@ -5,11 +5,11 @@ module Syskit class ProcessServerFixture attr_reader :loader - attr_reader :tasks + attr_reader :processes attr_reader :killed_processes def initialize @killed_processes = [] - @tasks = {} + @processes = {} @loader = FlexMock.undefined end @@ -20,7 +20,7 @@ def wait_termination(*) end def start(name, *) - tasks.fetch(name) + processes.fetch(name) end def wait_running(*process_names) @@ -37,23 +37,18 @@ def wait_running(*process_names) def disconnect; end end class ProcessFixture - - class ModelFixture - def extended_state_support? - false - end - end - attr_reader :process_server attr_reader :name_mappings attr_reader :ior_mappings - attr_reader :model attr_reader :property_names + attr_accessor :tasks + def initialize(process_server) @process_server = process_server @name_mappings = Hash["task" => "mapped_task_name"] @ior_mappings = { "mapped_task_name" => "IOR" } @property_names = [] + @tasks = {} end def get_mapped_name(name) @@ -69,9 +64,8 @@ def define_ior_mappings(ior_mappings) end def resolve_all_tasks - process_server.tasks + tasks end - end describe Deployment do @@ -86,13 +80,13 @@ def resolve_all_tasks @process_server = ProcessServerFixture.new @process = ProcessFixture.new(process_server) - process_server.tasks["mapped_task_name"] = process @log_dir = flexmock("log_dir") @process_server_config = Syskit.conf.register_process_server("fixture", process_server, log_dir) @deployment_task = deployment_m .new(process_name: "mapped_task_name", on: "fixture", name_mappings: { "task" => "mapped_task_name" }) + process_server.processes["mapped_task_name"] = process plan.add_permanent_task(@deployment_task) flexmock(process_server) @@ -353,11 +347,11 @@ def mock_raw_port(task, port_name) describe "monitoring for ready" do attr_reader :orocos_task before do - process_server.should_receive(:start).and_return(process) @orocos_task = Orocos.allow_blocking_calls do Orocos::RubyTasks::TaskContext.new "test" end - process.define_ior_mappings({"mapped_task_name" => "IOR"}) + process.tasks["test"] = @orocos_task + process_server.should_receive(:start).and_return(process) end after do orocos_task.dispose @@ -366,26 +360,58 @@ def mock_raw_port(task, port_name) it "does not emit ready if the process is not ready yet" do expect_execution { deployment_task.start! } .join_all_waiting_work(false).to_run - sync = Concurrent::Event.new - process.should_receive(:resolve_all_tasks) - .and_return do - sync.set - nil - end - sync.wait - expect_execution { sync.wait } + client_mock = flexmock(process_server_config.client) + client_mock + .should_receive(:wait_running) + .and_return({}) + expect_execution .join_all_waiting_work(false) .to { not_emit deployment_task.ready_event } end - it "is interrupted by the stop command" do + it "does not try resolving the task handles when resolve all" \ + "tasks raises an exception" do + flexmock(process) + .should_receive(:resolve_all_tasks) + .and_raise(Orocos::IORNotRegisteredError, "some error") expect_execution do deployment_task.start! - deployment_task.stop! end.to do - not_emit deployment_task.ready_event - emit deployment_task.stop_event + flexmock(execution_engine).should_receive(:promise).never + end + end + + it "does not emit_failed the ready event when the asynchronous part of "\ + "the ready codepath is interrupted by the stop command" do + expect_execution { deployment_task.start! } + .join_all_waiting_work(false) + .to { emit deployment_task.start_event } + + # Use a barrier to make sure we do the ready event + # processing after we actually stopped the deployment + barrier = Concurrent::CyclicBarrier.new(2) + flexmock(deployment_task) + .should_receive(:resolve_remote_task_handles) + .and_return do + barrier.wait + raise "fail the remote calls" + end + + # Wait for the deployment ready codepath to reach the barrier + expect_execution.join_all_waiting_work(false).to do + achieve { barrier.number_waiting == 1 } end + + plan.unmark_permanent_task(deployment_task) + expect_execution { deployment_task.stop! } + .join_all_waiting_work(false) + .to do + not_emit deployment_task.ready_event + emit deployment_task.stop_event + end + + barrier.wait + expect_execution.to_run end def make_deployment_ready @@ -412,34 +438,13 @@ def initialize_remote_handles(handles); end end it "polls for process readiness" do - sync = Concurrent::Event.new - process.should_receive(:resolve_all_tasks) - .and_return do - if !sync.set? - sync.set - nil - else Hash["mapped_task_name" => orocos_task] - end - end - expect_execution { deployment_task.start! } .join_all_waiting_work(false).to_run - sync.wait - expect_execution { sync.wait } + expect_execution + .poll { Syskit::Runtime.update_deployment_states(plan) } .to { emit deployment_task.ready_event } end - it "fails the ready event if the ready event monitor raises" do - expect_execution do - process.should_receive(:resolve_all_tasks) - .and_raise(RuntimeError.new("some message")) - deployment_task.start! - end.to do - have_error_matching( - Roby::EmissionFailed - .match.with_origin(deployment_task.ready_event) - ) - end - end + it "emits ready when the process is ready" do make_deployment_ready assert deployment_task.ready? @@ -724,7 +729,7 @@ def teardown def create_deployment(host_id, name: flexmock) process_server = ProcessServerFixture.new process = ProcessFixture.new(process_server) - process_server.tasks["mapped_task_name"] = process + process_server.processes["mapped_task_name"] = process log_dir = flexmock("log_dir") process_server_config = Syskit.conf.register_process_server(name, process_server, log_dir, host_id: host_id) From b3c44eb9194fe61f1e4045119195c0138aae5ccc Mon Sep 17 00:00:00 2001 From: Jhonas Date: Thu, 20 Oct 2022 10:51:53 -0300 Subject: [PATCH 009/260] fix(test): add update deployment state trigger ready tests --- test/runtime/test_update_deployment_state.rb | 189 ++++++++++++++++++- 1 file changed, 180 insertions(+), 9 deletions(-) diff --git a/test/runtime/test_update_deployment_state.rb b/test/runtime/test_update_deployment_state.rb index 3eb405e05..1695d714b 100644 --- a/test/runtime/test_update_deployment_state.rb +++ b/test/runtime/test_update_deployment_state.rb @@ -4,16 +4,187 @@ module Syskit module Runtime + class ProcessServerFixture + attr_reader :loader + attr_reader :processes + attr_reader :killed_processes + def initialize + @killed_processes = [] + @processes = {} + @loader = FlexMock.undefined + end + + def wait_termination(*) + dead_processes = @killed_processes + @killed_processes = [] + dead_processes + end + + def start(name, *) + processes.fetch(name) + end + + def wait_running(*process_names) + resolved = {} + process_names.each do |p_name| + resolved[p_name] = {} + @processes.each_value do |p| + resolved[p_name] = p.ior_mappings + end + end + resolved + end + + def disconnect; end + end + describe ".update_deployment_states" do - it "calls #dead! on the dead deployments" do - client = flexmock - flexmock(Syskit.conf).should_receive(:each_process_server_config) - .and_return([flexmock(client: client)]) - client.should_receive(:wait_termination).and_return([[p = flexmock, s = flexmock]]) - flexmock(Deployment).should_receive(:deployment_by_process).with(p) - .and_return(d = flexmock(finishing?: true)) - d.should_receive(:dead!).with(s).once - Runtime.update_deployment_states(plan) + describe "#handle_dead_deployments" do + it "calls #dead! on the dead deployments" do + client = flexmock + flexmock(Syskit.conf).should_receive(:each_process_server_config) + .and_return([flexmock(client: client)]) + client.should_receive(:wait_termination) + .and_return([[p = flexmock, s = flexmock]]) + flexmock(Deployment).should_receive(:deployment_by_process).with(p) + .and_return(d = flexmock(finishing?: true)) + d.should_receive(:dead!).with(s).once + Runtime.update_deployment_states(plan) + end + end + + describe "#trigger_ready_deployments" do + attr_reader :mocked_remote_tasks + attr_reader :process_server_config + + before do + @mocked_remote_tasks = { + "first_process" => { + iors: { + "task_name" => "IOR", + "task2" => "IOR2" + } + }, + "other_process" => { + iors: { + "third_task" => "IOR3", + "fourth_task" => "IOR4" + } + } + } + process_server = ProcessServerFixture.new + @process_server_config = Syskit.conf.register_process_server( + "fixture", process_server, flexmock("log_dir") + ) + end + + it "updates the deployments remote tasks when they are ready" do + client_mock = flexmock(process_server_config.client) + d1 = mocked_deployment( + "first_process", "fixture", mocked_remote_tasks["first_process"] + ) + d2 = mocked_deployment( + "other_process", "fixture", mocked_remote_tasks["other_process"] + ) + + flexmock(Runtime) + .should_receive(:find_all_not_ready_deployments) + .and_return({ "fixture": [d1, d2] }) + flexmock(Syskit.conf) + .should_receive(:process_server_config_for) + .and_return(process_server_config) + client_mock.should_receive(:wait_running) + .and_return(mocked_remote_tasks) + Runtime.update_deployment_states(plan) + end + + it "emits that the ready event failed and an error was received" do + mocked_remote_tasks["other_process"] = { error: "some error" } + client_mock = flexmock(process_server_config.client) + d1 = mocked_deployment( + "first_process", "fixture", mocked_remote_tasks["first_process"] + ) + d2 = mocked_deployment( + "other_process", "fixture", mocked_remote_tasks["other_process"] + ) + + flexmock(Runtime) + .should_receive(:find_all_not_ready_deployments) + .and_return({ "fixture": [d1, d2] }) + flexmock(Syskit.conf) + .should_receive(:process_server_config_for) + .and_return(process_server_config) + client_mock.should_receive(:wait_running) + .and_return(mocked_remote_tasks) + Runtime.update_deployment_states(plan) + end + + it "ignores the deployment when it hasnt received a result from "\ + "wait running" do + client_mock = flexmock(process_server_config.client) + d1 = mocked_deployment( + "first_process", "fixture", mocked_remote_tasks["first_process"] + ) + + flexmock(Runtime) + .should_receive(:find_all_not_ready_deployments) + .and_return({ "fixture": [d1] }) + flexmock(Syskit.conf) + .should_receive(:process_server_config_for) + .and_return(process_server_config) + client_mock.should_receive(:wait_running) + .and_return({ "first_process": nil }) + flexmock(d1.ready_event).should_receive(:emit_failed).never + flexmock(d1).should_receive(:update_remote_tasks).never + Runtime.update_deployment_states(plan) + end + + it "ignores the deployment when its ready event is pending" do + client_mock = flexmock(process_server_config.client) + d1 = mocked_deployment( + "first_process", "fixture", mocked_remote_tasks["first_process"], + pending: true + ) + + flexmock(Runtime) + .should_receive(:find_all_not_ready_deployments) + .and_return({ "fixture": [d1] }) + flexmock(Syskit.conf) + .should_receive(:process_server_config_for) + .and_return(process_server_config) + client_mock.should_receive(:wait_running) + .and_return({ + "first_process" => { + iors: { + "task_name" => "IOR", + "task2" => "IOR2" + } + } + }) + flexmock(d1.ready_event).should_receive(:emit_failed).never + flexmock(d1).should_receive(:update_remote_tasks).never + Runtime.update_deployment_states(plan) + end + + def mocked_deployment( + process_name, process_server_name, remote_tasks, pending: false + ) + ready_event = flexmock({ pending?: pending }) + mock = flexmock({ process_name: process_name, + arguments: { on: process_server_name }, + ready_event: ready_event }) + + error = remote_tasks[:error] + if error + mock.ready_event.should_receive(:emit_failed).with(error) + mock.should_receive(:update_remote_tasks).never + else + mock.should_receive(:ready_event).never + mock.should_receive(:update_remote_tasks) + .with(remote_tasks[:iors]) + end + mock + end end end end From d01fcd48830b81cd9a59b3407f03139384f72ad1 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Tue, 11 Oct 2022 10:25:16 -0300 Subject: [PATCH 010/260] fix(test): update the unit tests for unmanaged tasks with the wait running interface Since using the name_service to resolve the tasks is deprecated for ruby tasks and remote processes, it's better to move the tests where the name service is specified to unmanaged tasks unit tests, which is the only one still using the name_service --- test/roby_app/test_configuration.rb | 19 -- test/roby_app/test_unmanaged_tasks.rb | 289 +++++++++++++++----------- 2 files changed, 164 insertions(+), 144 deletions(-) diff --git a/test/roby_app/test_configuration.rb b/test/roby_app/test_configuration.rb index 49e66022a..cf017e99f 100644 --- a/test/roby_app/test_configuration.rb +++ b/test/roby_app/test_configuration.rb @@ -117,25 +117,6 @@ def stub_deployment(name) process = client.start "deployment", deployment_m.orogen_model assert_equal 20, process.pid end - - it "allows to specify the name service used to resolve the process' task" do - process_server_start - - name_service = Orocos::Local::NameService.new - name_service.register ruby_task, "resolved-remote-name" - client = @conf.connect_to_orocos_process_server( - "test-remote", "localhost", - port: process_server_port, - name_service: name_service - ) - - process = client.start( - "deployment", deployment_m.orogen_model, - { "name" => "resolved-remote-name" } - ) - tasks = process.resolve_all_tasks - assert_equal({ "resolved-remote-name" => ruby_task }, tasks) - end end def process_server_create diff --git a/test/roby_app/test_unmanaged_tasks.rb b/test/roby_app/test_unmanaged_tasks.rb index 9c1f4de02..a8a74e096 100644 --- a/test/roby_app/test_unmanaged_tasks.rb +++ b/test/roby_app/test_unmanaged_tasks.rb @@ -7,27 +7,176 @@ module RobyApp describe UnmanagedTasksManager do attr_reader :process_manager, :task_model, :unmanaged_task, :deployment_task - before do - @task_model = Syskit::TaskContext.new_submodel - Syskit.conf.register_process_server( - "unmanaged_tasks", UnmanagedTasksManager.new - ) - @process_manager = Syskit.conf.process_server_for("unmanaged_tasks") - use_unmanaged_task task_model => "unmanaged_deployment_test" - @task = syskit_deploy(task_model) - @deployment_task = @task.execution_agent - plan.unmark_mission_task(@task) - end + describe "using the default name service" do + before do + @task_model = Syskit::TaskContext.new_submodel + Syskit.conf.register_process_server( + "unmanaged_tasks", UnmanagedTasksManager.new + ) + @process_manager = Syskit.conf.process_server_for("unmanaged_tasks") + use_unmanaged_task task_model => "unmanaged_deployment_test" + @task = syskit_deploy(task_model) + @deployment_task = @task.execution_agent + plan.unmark_mission_task(@task) + end + + after do + if deployment_task.running? + expect_execution { deployment_task.stop! } + .to { emit deployment_task.stop_event } + end + + unmanaged_task&.dispose + + Syskit.conf.remove_process_server("unmanaged_tasks") + end + + it "sets the deployment's process name's to the specified name" do + assert_equal "unmanaged_deployment_test", deployment_task.process_name + end - after do - if deployment_task.running? + it "readies the execution agent when the task becomes available" do + expect_execution { deployment_task.start! } + .join_all_waiting_work(false) + .to do + emit deployment_task.start_event + not_emit deployment_task.ready_event + end + + create_unmanaged_task + expect_execution.to { emit deployment_task.ready_event } + task = deployment_task.task("unmanaged_deployment_test") + assert_equal unmanaged_task, task.orocos_task + end + + it "stopping the process causes the monitor thread to quit" do + make_deployment_ready + monitor_thread = deployment_task.orocos_process.monitor_thread expect_execution { deployment_task.stop! } .to { emit deployment_task.stop_event } + refute deployment_task.orocos_process.monitor_thread.alive? + assert deployment_task.orocos_process.dead? + refute monitor_thread.alive? end - unmanaged_task&.dispose + it "allows to kill a started deployment that was not ready" do + expect_execution { deployment_task.start! } + .join_all_waiting_work(false) + .to { emit deployment_task.start_event } + expect_execution { deployment_task.stop! } + .join_all_waiting_work(false) + .to { emit deployment_task.stop_event } + end - Syskit.conf.remove_process_server("unmanaged_tasks") + it "allows to kill a deployment that is ready" do + make_deployment_ready + expect_execution { deployment_task.stop! } + .to { emit deployment_task.stop_event } + end + + it "aborts the execution agent if the monitor thread fails "\ + "in unexpected ways" do + make_deployment_ready + + process_died = capture_log(deployment_task, :warn) do + background_thread_died = + capture_log(deployment_task.orocos_process, :fatal) do + expect_execution do + deployment_task + .orocos_process.monitor_thread + .raise RuntimeError + end.to { emit deployment_task.failed_event } + end + assert_equal ["assuming #{deployment_task.orocos_process} died "\ + "because the background thread died with", + "RuntimeError (RuntimeError)"], + background_thread_died + end + assert_equal ["unmanaged_deployment_test unexpectedly died "\ + "on process server unmanaged_tasks"], + process_died + end + + it "deregisters the process object of the process whose thread failed" do + make_deployment_ready + process = deployment_task.orocos_process + process_manager = process.process_manager + begin + flexmock(process).should_receive(dead?: true) + flexmock(process) + .should_receive(:verify_threads_state) + .and_raise(RuntimeError) + capture_log(process, :fatal) do + assert_equal [process], + process_manager.wait_termination(0).to_a + assert_equal Set[], process_manager.wait_termination(0) + end + ensure ( + process_manager.processes["unmanaged_deployment_test"] = process + ) + end + end + + it "stops the deployment if the remote task becomes unavailable" do + make_deployment_ready + messages = capture_log(deployment_task, :warn) do + expect_execution do + delete_unmanaged_task + # Synchronize on the monitor thread explicitely, otherwise + # the RTT state read might kick in first and bypass the + # whole purpose of the test + deployment_task.orocos_process.monitor_thread.join + assert deployment_task.orocos_process.dead? + end.to { emit deployment_task.failed_event } + end + assert_equal ["unmanaged_deployment_test unexpectedly died on "\ + "process server unmanaged_tasks"], + messages + end + + # This is really a heisentest .... previous versions of + # UnmanagedProcess would fail when this happened but the current + # implementation should be completely imprevious + it "handles concurrently having the monitor fail and #kill being called" \ + do + make_deployment_ready + expect_execution do + delete_unmanaged_task + deployment_task.orocos_process.kill + end.to { emit deployment_task.stop_event } + end + end + + describe "specifying the name service" do + attr_reader :name_service + before do + @task_model = Syskit::TaskContext.new_submodel + @name_service = Orocos::Local::NameService.new + Syskit.conf.register_process_server( + "new_unmanaged_tasks", + UnmanagedTasksManager.new(name_service: name_service) + ) + @process_manager = + Syskit.conf.process_server_for("new_unmanaged_tasks") + use_unmanaged_task(task_model => "unmanaged_deployment_test", + on: "new_unmanaged_tasks") + @task = syskit_deploy(task_model) + @deployment_task = @task.execution_agent + plan.unmark_mission_task(@task) + end + + it "allows to specify the name service used to resolve tasks" do + create_unmanaged_task + name_service.register @unmanaged_task + + process = process_manager.start( + "unmanaged_deployment_test", deployment_task.model.orogen_model + ) + result = process.wait_running + assert_includes(result, "unmanaged_deployment_test") + assert_kind_of(String, result["unmanaged_deployment_test"]) + assert_match(/^IOR:/, result["unmanaged_deployment_test"]) + end end def create_unmanaged_task @@ -51,116 +200,6 @@ def delete_unmanaged_task unmanaged_task.dispose @unmanaged_task = nil end - - it "sets the deployment's process name's to the specified name" do - assert_equal "unmanaged_deployment_test", deployment_task.process_name - end - - it "readies the execution agent when the task becomes available" do - expect_execution { deployment_task.start! } - .join_all_waiting_work(false) - .to do - emit deployment_task.start_event - not_emit deployment_task.ready_event - end - - create_unmanaged_task - expect_execution.to { emit deployment_task.ready_event } - task = deployment_task.task("unmanaged_deployment_test") - assert_equal unmanaged_task, task.orocos_task - end - - it "stopping the process causes the monitor thread to quit" do - make_deployment_ready - monitor_thread = deployment_task.orocos_process.monitor_thread - expect_execution { deployment_task.stop! } - .to { emit deployment_task.stop_event } - refute deployment_task.orocos_process.monitor_thread.alive? - assert deployment_task.orocos_process.dead? - refute monitor_thread.alive? - end - - it "allows to kill a started deployment that was not ready" do - expect_execution { deployment_task.start! } - .join_all_waiting_work(false) - .to { emit deployment_task.start_event } - expect_execution { deployment_task.stop! } - .join_all_waiting_work(false) - .to { emit deployment_task.stop_event } - end - - it "allows to kill a deployment that is ready" do - make_deployment_ready - expect_execution { deployment_task.stop! } - .to { emit deployment_task.stop_event } - end - - it "aborts the execution agent if the monitor thread fails "\ - "in unexpected ways" do - make_deployment_ready - - process_died = capture_log(deployment_task, :warn) do - background_thread_died = - capture_log(deployment_task.orocos_process, :fatal) do - expect_execution do - deployment_task - .orocos_process.monitor_thread - .raise RuntimeError - end.to { emit deployment_task.failed_event } - end - assert_equal ["assuming #{deployment_task.orocos_process} died "\ - "because the background thread died with", - "RuntimeError (RuntimeError)"], background_thread_died - end - assert_equal ["unmanaged_deployment_test unexpectedly died "\ - "on process server unmanaged_tasks"], - process_died - end - - it "deregisters the process object of the process whose thread failed" do - make_deployment_ready - process = deployment_task.orocos_process - process_manager = process.process_manager - begin - flexmock(process).should_receive(dead?: true) - flexmock(process) - .should_receive(:verify_threads_state) - .and_raise(RuntimeError) - capture_log(process, :fatal) do - assert_equal [process], process_manager.wait_termination(0).to_a - assert_equal Set[], process_manager.wait_termination(0) - end - ensure process_manager.processes["unmanaged_deployment_test"] = process - end - end - - it "stops the deployment if the remote task becomes unavailable" do - make_deployment_ready - messages = capture_log(deployment_task, :warn) do - expect_execution do - delete_unmanaged_task - # Synchronize on the monitor thread explicitely, otherwise - # the RTT state read might kick in first and bypass the - # whole purpose of the test - deployment_task.orocos_process.monitor_thread.join - assert deployment_task.orocos_process.dead? - end.to { emit deployment_task.failed_event } - end - assert_equal ["unmanaged_deployment_test unexpectedly died on "\ - "process server unmanaged_tasks"], - messages - end - - # This is really a heisentest .... previous versions of - # UnmanagedProcess would fail when this happened but the current - # implementation should be completely imprevious - it "handles concurrently having the monitor fail and #kill being called" do - make_deployment_ready - expect_execution do - delete_unmanaged_task - deployment_task.orocos_process.kill - end.to { emit deployment_task.stop_event } - end end end end From ca5cf940de05c3eae6c75b193a0c19d746c81bb2 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Thu, 20 Oct 2022 11:07:46 -0300 Subject: [PATCH 011/260] fix(test): update task context tests --- test/test_task_context.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_task_context.rb b/test/test_task_context.rb index 08e6e1f83..e87705a13 100644 --- a/test/test_task_context.rb +++ b/test/test_task_context.rb @@ -1750,6 +1750,7 @@ def configure_tasks @remote_test_property = task.orocos_task.raw_property("test") @remote_test_property.write(0.2) end + @property.update_remote_value(0.2) end def mock_remote_property property.remote_property = @remote_test_property @@ -1830,10 +1831,10 @@ def mock_remote_property @property = task.test_property Orocos.allow_blocking_calls do @remote_test_property = task.orocos_task.raw_property("test") - @test_property_initial_value = @remote_test_property.read @remote_test_property.write(0.2) end end + def mock_remote_property property.remote_property = @remote_test_property flexmock(@remote_test_property) @@ -1922,6 +1923,7 @@ def task.configure @remote_test_property = task.orocos_task.raw_property("test") remote_test_property.write(0.2) end + @stub_property.update_remote_value(0.2) @guard = syskit_guard_against_start_and_configure end def mock_remote_property @@ -1988,7 +1990,7 @@ def mock_remote_property end promises.each(&:execute) execution_engine.join_all_waiting_work - assert_equal (0...100).to_a, finished + assert_equal (0...100).to_a, finished.sort end it "is serialized with the initial commit in task setup" do From d5a89960f2509fd838b1c2edd0eac33098815779 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 27 Oct 2022 14:35:18 -0300 Subject: [PATCH 012/260] fix: change task name in process fixture The task name was not the expected one. The error was "hidden" because of the bug fixed by https://github.com/rock-core/tools-roby/pull/230 --- test/test_deployment.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_deployment.rb b/test/test_deployment.rb index f33c32c18..90acb7f78 100644 --- a/test/test_deployment.rb +++ b/test/test_deployment.rb @@ -350,7 +350,7 @@ def mock_raw_port(task, port_name) @orocos_task = Orocos.allow_blocking_calls do Orocos::RubyTasks::TaskContext.new "test" end - process.tasks["test"] = @orocos_task + process.tasks["mapped_task_name"] = @orocos_task process_server.should_receive(:start).and_return(process) end after do From 01436b70da833245a57670fcc98ec72b66080c59 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Tue, 22 Nov 2022 14:51:38 -0300 Subject: [PATCH 013/260] feat: add policy to model level component data readers and writers --- lib/syskit/models/component.rb | 8 ++++---- test/test_component.rb | 22 ++++++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/syskit/models/component.rb b/lib/syskit/models/component.rb index 00e3aa9ba..7602f5cd4 100644 --- a/lib/syskit/models/component.rb +++ b/lib/syskit/models/component.rb @@ -1311,14 +1311,14 @@ def match # data_reader some_child.out_port, as: 'pose' # # @return [DynamicPortBinding::BoundOutputReader] - def data_reader(port, as:) + def data_reader(port, as:, **policy) port = DynamicPortBinding.create(port) unless port.output? raise ArgumentError, "expected an output port, but #{port} seems to be an input" end - data_readers[as] = port.to_bound_data_accessor(as, self) + data_readers[as] = port.to_bound_data_accessor(as, self, **policy) end # The data writers defined on this task, as a mapping from the writer's @@ -1344,14 +1344,14 @@ def data_reader(port, as:) # data_reader some_child.cmd_in_port, as: 'cmd_in' # # @return [DynamicPortBinding::BoundInputWriter] - def data_writer(port, as:) + def data_writer(port, as:, **policy) port = DynamicPortBinding.create(port) if port.output? raise ArgumentError, "expected an input port, but #{port} seems to be an output" end - data_writers[as] = port.to_bound_data_accessor(as, self) + data_writers[as] = port.to_bound_data_accessor(as, self, **policy) end def has_through_method_missing?(name) diff --git a/test/test_component.rb b/test/test_component.rb index d302b1a74..dc3e12948 100644 --- a/test/test_component.rb +++ b/test/test_component.rb @@ -113,8 +113,8 @@ before do @task_m = Syskit::TaskContext.new_submodel do output_port "out", "int" - dynamic_output_port /\w+_out/, "bool" - dynamic_input_port /\w+_in/, "double" + dynamic_output_port(/\w+_out/, "bool") + dynamic_input_port(/\w+_in/, "double") end srv_m = @srv_m = Syskit::DataService.new_submodel do output_port "out", "bool" @@ -512,7 +512,7 @@ it "creates dynamic ports" do task_m = Syskit::TaskContext.new_submodel do - dynamic_output_port /\w+/, nil + dynamic_output_port(/\w+/, nil) end dynport = task_m.orogen_model.dynamic_ports.find { true } @@ -645,6 +645,13 @@ assert_equal task.out_port, reader.resolved_accessor.port end + it "creates a bound accessor with the specified policy" do + reader = @support_task_m.data_reader( + @task_m.out_port, type: :buffer, size: 20, as: "test" + ) + assert_equal({ type: :buffer, size: 20 }, reader.policy) + end + it "enumerates the bound readers with each_data_reader" do @support_task_m.data_reader @task_m.match.out_port, as: "test" support_task = syskit_stub_and_deploy(@support_task_m) @@ -914,6 +921,13 @@ assert_equal task.in_port, reader.resolved_accessor.port end + it "creates a bound accessor with the specified policy" do + writer = @support_task_m.data_writer( + @task_m.in_port, type: :buffer, size: 20, as: "test" + ) + assert_equal({ type: :buffer, size: 20 }, writer.policy) + end + it "enumerates the bound writers with each_data_writer" do @support_task_m.data_writer @task_m.match.in_port, as: "test" support_task = syskit_stub_and_deploy(@support_task_m) @@ -1172,7 +1186,7 @@ def test_disconnect_ports assert(source_task.connected_to?("out", sink_task, "out")) assert(source_task.connected_to?("out", sink_task, "other")) - source_task.disconnect_ports(sink_task, [%w{out other}]) + source_task.disconnect_ports(sink_task, [%w[out other]]) assert_equal( { %w[out out] => { type: :buffer, size: 20 } From cd6d27cc6b605763d9effbb17e8ba0d572343bc0 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 3 Nov 2022 17:53:29 -0300 Subject: [PATCH 014/260] feat: implement rate-liming in log transfer --- .../roby_app/remote_processes/client.rb | 8 +- .../roby_app/remote_processes/ftp_upload.rb | 99 +++++++++++++++++++ .../remote_processes/log_upload_state.rb | 6 -- .../roby_app/remote_processes/server.rb | 39 ++------ .../remote_processes/test_remote_processes.rb | 27 ++++- 5 files changed, 136 insertions(+), 43 deletions(-) create mode 100644 lib/syskit/roby_app/remote_processes/ftp_upload.rb diff --git a/lib/syskit/roby_app/remote_processes/client.rb b/lib/syskit/roby_app/remote_processes/client.rb index d99db4952..31da6cdb6 100644 --- a/lib/syskit/roby_app/remote_processes/client.rb +++ b/lib/syskit/roby_app/remote_processes/client.rb @@ -230,10 +230,14 @@ def queue_death_announcement # # The transfer is asynchronous, use {#upload_state} to track the # upload progress - def log_upload_file(host, port, certificate, user, password, localfile) + def log_upload_file( + host, port, certificate, user, password, localfile, + max_upload_rate: Float::INFINITY + ) socket.write(COMMAND_LOG_UPLOAD_FILE) Marshal.dump( - [host, port, certificate, user, password, localfile], socket + [host, port, certificate, user, password, localfile, + max_upload_rate], socket ) wait_for_ack diff --git a/lib/syskit/roby_app/remote_processes/ftp_upload.rb b/lib/syskit/roby_app/remote_processes/ftp_upload.rb new file mode 100644 index 000000000..4bbc68d42 --- /dev/null +++ b/lib/syskit/roby_app/remote_processes/ftp_upload.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Syskit + module RobyApp + module RemoteProcesses + # Encapsulation of the log file upload process + class FTPUpload + def initialize( # rubocop:disable Metrics/ParameterLists + host, port, certificate, user, password, file, + max_upload_rate: Float::INFINITY + ) + + @host = host + @port = port + @certificate = certificate + @user = user + @password = password + @file = file + + @max_upload_rate = Float(max_upload_rate) + end + + Result = Struct.new :file, :success, :message do + def success? + success + end + end + + # Create a temporary file with the FTP server's public key, to pass + # to FTP.open + # + # @yieldparam [String] path the certificate path + def with_certificate + Tempfile.create do |cert_io| + cert_io.write @certificate + cert_io.flush + yield(cert_io.path) + end + end + + # Open the FTP connection + # + # @yieldparam [Net::FTP] + def open + with_certificate do |cert_path| + Net::FTP.open( + @host, + private_data_connection: false, port: @port, + ssl: { verify_mode: OpenSSL::SSL::VERIFY_PEER, + ca_file: cert_path } + ) do |ftp| + ftp.login(@user, @password) + yield(ftp) + end + end + end + + # Open the connection and transfer the file + # + # @return [UploadResult] + def open_and_transfer + open { |ftp| transfer(ftp) } + Result.new(@file, true, nil) + rescue StandardError => e + Result.new(@file, false, e.message) + end + + # Do transfer the file through the given connection + # + # @param [Net::FTP] ftp + def transfer(ftp) + last = Time.now + File.open(@file) do |file_io| + ftp.storbinary("STOR #{File.basename(@file)}", + file_io, Net::FTP::DEFAULT_BLOCKSIZE) do |buf| + now = Time.now + rate_limit(buf.size, now, last) + last = Time.now + end + end + end + + # @api private + # + # Sleep when needed to keep the expected transfer rate + def rate_limit(chunk_size, now, last) + duration = now - last + exp_duration = chunk_size / @max_upload_rate + # Do not wait, but do not try to "make up" for the bandwidth + # we did not use. The goal is to not affect the rest of the + # system + return if duration > exp_duration + + sleep(exp_duration - duration) + end + end + end + end +end diff --git a/lib/syskit/roby_app/remote_processes/log_upload_state.rb b/lib/syskit/roby_app/remote_processes/log_upload_state.rb index 88599b627..da659b2f0 100644 --- a/lib/syskit/roby_app/remote_processes/log_upload_state.rb +++ b/lib/syskit/roby_app/remote_processes/log_upload_state.rb @@ -5,12 +5,6 @@ module RobyApp module RemoteProcesses # State of the asynchronous file transfers managed by {Server} class LogUploadState - Result = Struct.new :file, :success, :message do - def success? - success - end - end - attr_reader :pending_count def initialize(pending_count, results) diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 9779d6cf1..2746960d5 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -7,6 +7,7 @@ require "concurrent/atomic/atomic_reference" require "syskit/roby_app/remote_processes/log_upload_state" +require "syskit/roby_app/remote_processes/ftp_upload" module Syskit module RobyApp @@ -405,13 +406,15 @@ def handle_command(socket) # :nodoc: elsif cmd_code == COMMAND_QUIT quit elsif cmd_code == COMMAND_LOG_UPLOAD_FILE - host, port, certificate, user, password, localfile = + host, port, certificate, user, password, localfile, + max_upload_rate = Marshal.load(socket) Server.debug "#{socket} requested uploading of #{localfile}" @log_upload_command_queue << - Upload.new( + FTPUpload.new( host, port, certificate, - user, password, localfile + user, password, localfile, + max_upload_rate: max_upload_rate || Float::INFINITY ) socket.write(RET_YES) @@ -571,38 +574,10 @@ def quit @com_w&.write INTERNAL_QUIT end - Upload = Struct.new( - :host, :port, :certificate, :user, :password, :file - ) do - def apply - Tempfile.create do |cert_io| - cert_io.write certificate - cert_io.flush - - Net::FTP.open( - host, - private_data_connection: false, - port: port, - ssl: { verify_mode: OpenSSL::SSL::VERIFY_PEER, - ca_file: cert_io.path } - ) do |ftp| - ftp.login(user, password) - File.open(file) do |file_io| - ftp.storbinary("STOR #{File.basename(file)}", - file_io, Net::FTP::DEFAULT_BLOCKSIZE) - end - end - end - LogUploadState::Result.new(file, true, nil) - rescue Exception => e - LogUploadState::Result.new(file, false, e.message) - end - end - def log_upload_main while (transfer = @log_upload_command_queue.pop) @log_upload_current.set(transfer) - @log_upload_results_queue << transfer.apply + @log_upload_results_queue << transfer.open_and_transfer @log_upload_current.set(nil) end end diff --git a/test/roby_app/remote_processes/test_remote_processes.rb b/test/roby_app/remote_processes/test_remote_processes.rb index 8fcbacd36..e8e91de7f 100644 --- a/test/roby_app/remote_processes/test_remote_processes.rb +++ b/test/roby_app/remote_processes/test_remote_processes.rb @@ -308,7 +308,7 @@ @port, @certificate = spawn_log_transfer_server @logfile = File.join(make_tmpdir, "logfile.log") File.open(@logfile, "wb") do |f| - f.write(SecureRandom.random_bytes(547)) # create random 5 MB file + f.write(SecureRandom.random_bytes(547)) # create random 547 byte end end @@ -328,6 +328,27 @@ assert_equal File.read(path), File.read(@logfile) end + it "rate-limits the file transfer" do + File.open(@logfile, "wb") do |f| + f.write(SecureRandom.random_bytes(1024 * 1024)) + end + tic = Time.now + client.log_upload_file( + "localhost", @port, @certificate, + @user, @password, @logfile, + max_upload_rate: 500 * 1024 + ) + assert_upload_succeeds(timeout: 5) + toc = Time.now + + assert_includes( + (1.8..2.2), toc - tic, + "transfer took #{toc - tic} instead of the expected 2s" + ) + path = File.join(@temp_serverdir, "logfile.log") + assert_equal File.read(path), File.read(@logfile) + end + it "rejects a wrong user" do client.log_upload_file( "localhost", @port, @certificate, @@ -406,8 +427,8 @@ def wait_for_upload_completion(poll_period: 0.01, timeout: 1) end end - def assert_upload_succeeds - wait_for_upload_completion.each_result do |r| + def assert_upload_succeeds(timeout: 1) + wait_for_upload_completion(timeout: timeout).each_result do |r| flunk("upload failed: #{r.message}") unless r.success? end end From cf9ac22aa37ca10f39bc2d751920061b880764ea Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 28 Nov 2022 14:25:35 -0300 Subject: [PATCH 015/260] fix: cleanup how one should configure the log upload functionality --- lib/syskit/roby_app/configuration.rb | 67 +++++++--- lib/syskit/roby_app/plugin.rb | 114 ++++++++++-------- .../roby_app/remote_processes/protocol.rb | 2 +- .../roby_app/remote_processes/server.rb | 3 +- 4 files changed, 114 insertions(+), 72 deletions(-) diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index c9a165532..e5e9f2d0e 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -49,6 +49,46 @@ class Configuration # is 20s attr_accessor :exception_transition_timeout + LogTransfer = Struct.new( + :enabled, :ip, :port, :user, :password, :certificate, + :self_spawned, :target_dir, keyword_init: true + ) do + def enabled? + enabled + end + + def self_spawned? + self_spawned + end + end + + # Configuration of Syskit's log transfer functionality + # + # Minimum configuration: set `ip` to an IP which the process servers + # can reach and set `enabled` to true. You must also configure log rotation + # ({#log_rotation_period}). Syskit will transfer the rotated logs to the + # main Syskit's instance log directory. + # + # If you want to transfer to another dir, also set {#target_dir}. If you do + # set {#target_dir}, local files will also be transferred. There is currently + # no optimization for this (the local logs will also be transferred through + # the network) + # + # If you want to use an external server, you must also provide its public + # certificate and set self_spawned to false. In this case, target_dir is + # ignored + # + # @return [LogTransfer] + attr_reader :log_transfer + + # Period in seconds for triggering log rotation and transfer + # + # This is considered experimental, and is disabled by default + # + # @return [Number] the rotation in seconds, or nil if rotation is + # disabled altogether + attr_accessor :log_rotation_period + # @deprecated Unused, kept here for historical reasons attr_predicate :ignore_load_errors, true # @deprecated Unused, kept here for historical reasons @@ -76,14 +116,6 @@ class Configuration # @return [Models::DeploymentGroup] attr_reader :deployment_group - # Period in seconds for triggering log rotation and transfer - # - # This is considered experimental, and is disabled by default - # - # @return [Number] the rotation in seconds, or nil if rotation is - # disabled altogether - attr_accessor :log_rotation_period - # Whether Syskit should instruct a process server to kill all its # processes on connection # @@ -94,15 +126,6 @@ class Configuration # likely want this attr_predicate :kill_all_on_process_server_connection?, true - # Whether the rotated logs should be uploaded to Syskit's local log dir - # - # This is considered experimental, and is disabled by default - def log_upload? - @log_upload - end - - attr_writer :log_upload - # Controls whether the orogen types should be exported as Ruby # constants # @@ -135,7 +158,15 @@ def initialize(app) @kill_all_on_process_server_connection = false @log_rotation_period = nil - @log_upload = false + @log_transfer = LogTransfer.new( + enabled: false, + user: "syskit", + port: 22, + password: SecureRandom.base64(32), + self_spawned: true, + certificate: nil, # Use random generated self-signed certificate + target_dir: nil # Use the app's log dir + ) clear diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index f68b1536b..de0b0c8e6 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -36,7 +36,6 @@ module RobyApp # a shell user module Plugin attr_writer :syskit_use_update_properties - attr_accessor :log_transfer_ip # Assume all component models have been migrated to use update_properties # @@ -161,57 +160,78 @@ def self.setup(app) Syskit::TaskContext.define_from_orogen(rtt_core_model, register: true) # Log Transfer FTP Server spawned during Application#setup - app.setup_local_log_transfer_server if app.log_transfer_ip - end + log_transfer = Syskit.conf.log_transfer + if log_transfer.enabled? + log_transfer.target_dir ||= Roby.app.log_dir - def setup_local_log_transfer_server - @log_transfer_user = "process server" - @log_transfer_password = SecureRandom.base64(15) - @tmp_root_ca = TmpRootCA.new(log_transfer_ip) + unless Syskit.conf.log_rotation_period + raise ArgumentError, + "cannot set up log transfer without log rotation" + end - start_local_log_transfer_server(log_dir, @log_transfer_user, @log_transfer_password, @tmp_root_ca.private_certificate_path) + app.log_transfer_start_server if log_transfer.self_spawned? + end end - def send_file_transfer_command(name, logfile) - unless @log_transfer_server - raise "log transfer server is not started" + def log_transfer_start_server + if @log_transfer_server + raise ArgumentError, "log transfer server already running" end - # Establishes communication with said process server - client = Syskit.conf.process_server_for(name) + conf = Syskit.conf.log_transfer + @log_transfer_ca = TmpRootCA.new(conf.ip) + conf.certificate = @log_transfer_ca.certificate - # Commands method log_upload_file from said process server - client.log_upload_file( - log_transfer_ip, - @log_transfer_server.port, - @tmp_root_ca.certificate, - @log_transfer_user, - @log_transfer_password, - logfile + @log_transfer_server = LogTransferServer::SpawnServer.new( + conf.target_dir, conf.user, conf.password, + @log_transfer_ca.private_certificate_path ) - client + conf.port = @log_transfer_server.port end - def start_local_log_transfer_server(tgt_dir, user, password, private_certificate_path) - if @log_transfer_server - raise "log transfer server is already started" - end + # Transfer the given files to the FTP server configured in + # {Configuration#log_transfer} + # + # @param [{String=>[String]}] logs mapping of server name to the list of + # log names to transfer for this server + def log_transfer_upload(logs) + logs.each do |process_server_name, process_server_logs| + process_server = + Syskit.conf.process_server_config_for(process_server_name) + conf = Syskit.conf.log_transfer + + if process_server.in_process? || process_server.on_localhost? + next if conf.target_dir == Roby.app.log_dir + end - @log_transfer_server = Syskit::RobyApp::LogTransferServer::SpawnServer.new( - tgt_dir, - user, - password, - private_certificate_path - ) + log_transfer_send_files(process_server_name, process_server_logs) + end end - def stop_local_log_transfer_server - if @log_transfer_server - @log_transfer_server.stop - @log_transfer_server.join - @log_transfer_server = nil + def log_transfer_send_files(name, logfiles) + raise "log transfer server is not started" unless @log_transfer_server + + conf = Syskit.conf.log_transfer + client = Syskit.conf.process_server_for(name) + logfiles.each do |path| + client.log_upload_file( + conf.ip, conf.port, conf.certificate, + conf.user, conf.password, path + ) end - @tmp_root_ca&.dispose + client + end + + def log_transfer_server_started? + @log_transfer_server + end + + def log_transfer_stop_server + @log_transfer_server.stop + @log_transfer_server.join + @log_transfer_ca.dispose + @log_transfer_server = nil + @log_transfer_ca = nil end # Hook called by the main application in Application#setup after @@ -245,7 +265,7 @@ def self.cleanup(app) disconnect_all_process_servers stop_local_process_server(app) - app.stop_local_log_transfer_server + app.log_transfer_stop_server if app.log_transfer_server_started? end # Hook called by the main application to prepare for execution @@ -255,7 +275,9 @@ def self.prepare(app) if Syskit.conf.log_rotation_period app.execution_engine.every(Syskit.conf.log_rotation_period) do rotated_logs = app.rotate_logs - app.upload_rotated_logs(rotated_logs) if Syskit.conf.log_upload? + if Syskit.conf.log_transfer.enabled? + app.log_transfer_upload(rotated_logs) + end end end end @@ -982,18 +1004,6 @@ def rotate_logs (rotated_logs[process_server_name] ||= []).concat(task.rotate_log) end end - - def upload_rotated_logs(rotated_logs) - rotated_logs.each do |process_server_name, logs| - process_server = Syskit.conf.process_server_config_for(process_server_name) - - if !process_server.in_process? && !process_server.on_localhost? - logs.each do |log_filename| - send_file_transfer_command(process_server_name, log_filename) - end - end - end - end end end end diff --git a/lib/syskit/roby_app/remote_processes/protocol.rb b/lib/syskit/roby_app/remote_processes/protocol.rb index 8e208ea47..5101856e7 100644 --- a/lib/syskit/roby_app/remote_processes/protocol.rb +++ b/lib/syskit/roby_app/remote_processes/protocol.rb @@ -13,7 +13,7 @@ module RemoteProcesses COMMAND_QUIT = "Q" COMMAND_KILL_ALL = "K" COMMAND_WAIT_RUNNING = "W" - COMMAND_LOG_UPLOAD_FILE = "U" + COMMAND_LOG_UPLOAD_FILE = "U" COMMAND_LOG_UPLOAD_STATE = "X" EVENT_DEAD_PROCESS = "D" diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 2746960d5..50bdd9003 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -304,6 +304,8 @@ def announce_dead_processes(dead_processes) # Helper method that stops all running processes def quit_and_join # :nodoc: + @log_upload_command_queue << nil + Server.info "stopping process server" processes.each_value do |p| Server.info "killing #{p.name}" @@ -318,7 +320,6 @@ def quit_and_join # :nodoc: rescue SystemCallError, IOError # rubocop:disable Lint/SuppressedException end - @log_upload_command_queue << nil @log_upload_thread.join end From e38e3aea12669a0ebc7bbab4fab39d6663ba34ec Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 28 Nov 2022 14:52:31 -0300 Subject: [PATCH 016/260] fix: switch log upload to implicit FTPs Meaning, both the server and client start in TLS mode. Without this, there is some exchange "in the clear" before they switch. I made this change not for the sake of security, but because the explicit mode started failing for no discernable reason. --- .../log_transfer_server/spawn_server.rb | 2 +- .../roby_app/remote_processes/ftp_upload.rb | 1 + .../spawn_server/test_spawn_server.rb | 210 +++++++++--------- 3 files changed, 109 insertions(+), 104 deletions(-) diff --git a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb index fd8f85a4c..eaa21c483 100644 --- a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb +++ b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb @@ -16,7 +16,7 @@ def initialize( password, certfile_path, interface: "127.0.0.1", - tls: :explicit, + tls: :implicit, port: 0, session_timeout: default_session_timeout, nat_ip: nil, diff --git a/lib/syskit/roby_app/remote_processes/ftp_upload.rb b/lib/syskit/roby_app/remote_processes/ftp_upload.rb index 4bbc68d42..66023ae39 100644 --- a/lib/syskit/roby_app/remote_processes/ftp_upload.rb +++ b/lib/syskit/roby_app/remote_processes/ftp_upload.rb @@ -46,6 +46,7 @@ def open Net::FTP.open( @host, private_data_connection: false, port: @port, + implicit_ftps: true, ssl: { verify_mode: OpenSSL::SSL::VERIFY_PEER, ca_file: cert_path } ) do |ftp| diff --git a/test/roby_app/spawn_server/test_spawn_server.rb b/test/roby_app/spawn_server/test_spawn_server.rb index d8e1ef4c6..ad89e784c 100644 --- a/test/roby_app/spawn_server/test_spawn_server.rb +++ b/test/roby_app/spawn_server/test_spawn_server.rb @@ -4,113 +4,117 @@ require "syskit/roby_app/log_transfer_server" require "net/ftp" -describe Syskit::RobyApp::LogTransferServer::SpawnServer do - class TestServer < Syskit::RobyApp::LogTransferServer::SpawnServer - attr_accessor :user, :password, :certfile_path - - def initialize(target_dir, user, password) - @certfile_path = File.join(__dir__, "..", "remote_processes", "cert.crt") - private_key_path = File.join( - __dir__, "..", "remote_processes", "cert-private.crt" - ) - super(target_dir, user, password, private_key_path) - @user = user - @password = password - end - end - - ### AUXILIARY FUNCTIONS ### - def spawn_server - @temp_serverdir = make_tmpdir - @user = "test.user" - @password = "test.password" - @server = TestServer.new(@temp_serverdir, @user, @password) - end - - def ftp_open(certfile_path: @server.certfile_path, &block) - Net::FTP.open( - "localhost", - port: @server.port, - ssl: { verify_mode: OpenSSL::SSL::VERIFY_PEER, verify_hostname: false, - ca_file: certfile_path }, - &block - ) - end - - def upload_log(user, password, localfile, certfile_path: @server.certfile_path) - ftp_open(certfile_path: certfile_path) do |ftp| - ftp.login(user, password) - File.open(localfile) do |lf| - ftp.storbinary("STOR #{File.basename(localfile)}", - lf, Net::FTP::DEFAULT_BLOCKSIZE) - end - end - end - - def upload_testfile - File.open(File.join(@temp_srcdir, "testfile"), "w+") do |tf| - upload_log(@server.user, @server.password, tf) - end - end - - ### TESTS ### - describe "#LogTransferServerTests" do - before do - spawn_server - @temp_srcdir = make_tmpdir - end - - after do - @server.stop - @server.join - end - - it "logs in successfully with the correct user and password" do - ftp_open do |ftp| - # Raises on error - ftp.login(@server.user, @server.password) - end - end - - it "rejects an invalid user" do - ftp_open do |ftp| - assert_raises(Net::FTPPermError) { ftp.login("user", @server.password) } - end - end - - it "rejects an invalid password" do - ftp_open do |ftp| - assert_raises(Net::FTPPermError) { ftp.login(@server.user, "password") } - end - end - - it "refuses to connect if the server's certificate is unexpected" do - invalid_certfile_path = File.join( - __dir__, "..", "remote_processes", "invalid-cert.crt" - ) +module Syskit + module RobyApp + module LogTransferServer + describe SpawnServer do + ### AUXILIARY FUNCTIONS ### + def spawn_server + @temp_serverdir = make_tmpdir + @user = "test.user" + @password = "test.password" + @certfile_path = + File.join(__dir__, "..", "remote_processes", "cert.crt") + private_key_path = File.join( + __dir__, "..", "remote_processes", "cert-private.crt" + ) + @server = SpawnServer.new( + @temp_serverdir, @user, @password, + private_key_path + ) + end - e = assert_raises(OpenSSL::SSL::SSLError) do - ftp_open(certfile_path: invalid_certfile_path) - end - assert_match(/certificate verify failed/, e.message) - end + def ftp_open(certfile_path: @certfile_path, &block) + Net::FTP.open( + "localhost", + port: @server.port, + implicit_ftps: true, + ssl: { verify_mode: OpenSSL::SSL::VERIFY_PEER, + verify_hostname: false, + ca_file: certfile_path }, + &block + ) + end - it "uploads a file to the server's directory" do - upload_testfile - assert File.exist?("#{@temp_serverdir}/testfile") - end + def upload_log(user, password, localfile, certfile_path: @certfile_path) + ftp_open(certfile_path: certfile_path) do |ftp| + ftp.login(user, password) + File.open(localfile) do |lf| + ftp.storbinary( + "STOR #{File.basename(localfile)}", + lf, Net::FTP::DEFAULT_BLOCKSIZE + ) + end + end + end - it "refuses to upload a file that already exists" do - upload_testfile - assert_raises(Net::FTPPermError) { upload_testfile } - end + def upload_testfile + File.open(File.join(@temp_srcdir, "testfile"), "w+") do |tf| + upload_log(@user, @password, tf) + end + end - it "refuses to GET a file" do - upload_testfile - ftp_open do |ftp| - ftp.login(@server.user, @server.password) - assert_raises(Net::FTPPermError) do - ftp.get("#{@temp_serverdir}/testfile") + ### TESTS ### + describe "#LogTransferServerTests" do + before do + spawn_server + @temp_srcdir = make_tmpdir + end + + after do + @server.stop + @server.join + end + + it "logs in successfully with the correct user and password" do + ftp_open do |ftp| + # Raises on error + ftp.login(@user, @password) + end + end + + it "rejects an invalid user" do + ftp_open do |ftp| + assert_raises(Net::FTPPermError) { ftp.login("user", @password) } + end + end + + it "rejects an invalid password" do + ftp_open do |ftp| + assert_raises(Net::FTPPermError) { ftp.login(@user, "password") } + end + end + + it "refuses to connect if the server's certificate is unexpected" do + invalid_certfile_path = File.join( + __dir__, "..", "remote_processes", "invalid-cert.crt" + ) + + e = assert_raises(OpenSSL::SSL::SSLError) do + ftp_open(certfile_path: invalid_certfile_path) + end + assert_match(/certificate verify failed/, e.message) + end + + it "uploads a file to the server's directory" do + upload_testfile + assert File.exist?("#{@temp_serverdir}/testfile") + end + + it "refuses to upload a file that already exists" do + upload_testfile + assert_raises(Net::FTPPermError) { upload_testfile } + end + + it "refuses to GET a file" do + upload_testfile + ftp_open do |ftp| + ftp.login(@user, @password) + assert_raises(Net::FTPPermError) do + ftp.get("#{@temp_serverdir}/testfile") + end + end + end end end end From a4547039839eddeaa02e47dd63740614063c908c Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 28 Nov 2022 15:56:22 -0300 Subject: [PATCH 017/260] fix: raise an error if log transfer is enabled but IP is not set --- lib/syskit/roby_app/plugin.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index de0b0c8e6..eabfa349e 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -162,6 +162,11 @@ def self.setup(app) # Log Transfer FTP Server spawned during Application#setup log_transfer = Syskit.conf.log_transfer if log_transfer.enabled? + unless log_transfer.ip + raise ArgumentError, + "log transfer is enabled, but log_transfer.ip is not set" + end + log_transfer.target_dir ||= Roby.app.log_dir unless Syskit.conf.log_rotation_period From 10316eb5c3c7013af3e0b4f9a4f6dbb0569f0671 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 28 Nov 2022 15:56:54 -0300 Subject: [PATCH 018/260] fix: flush log transfers on quit --- lib/syskit/roby_app/plugin.rb | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index eabfa349e..390c02676 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -231,6 +231,22 @@ def log_transfer_server_started? @log_transfer_server end + def log_transfer_flush + clients = Syskit.conf.each_process_server.to_a + until clients.empty? + clients = clients.find_all do |c| + state = c.log_upload_state + next(false) if state.pending == 0 + + Robot.info "Waiting for process server at #{c.host} "\ + "to finish uploading" + true + end + + sleep(0.5) unless clients.empty? + end + end + def log_transfer_stop_server @log_transfer_server.stop @log_transfer_server.join @@ -270,7 +286,11 @@ def self.cleanup(app) disconnect_all_process_servers stop_local_process_server(app) - app.log_transfer_stop_server if app.log_transfer_server_started? + + if app.log_transfer_server_started? + app.log_transfer_flush + app.log_transfer_stop_server + end end # Hook called by the main application to prepare for execution From 5640ad024155ccfb414685dcbc4373a43106a794 Mon Sep 17 00:00:00 2001 From: caioaamaral Date: Mon, 24 Oct 2022 16:12:18 -0800 Subject: [PATCH 019/260] chore(net_gen): remove extra ')' from a warn message --- lib/syskit/network_generation/logger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syskit/network_generation/logger.rb b/lib/syskit/network_generation/logger.rb index c7f7ced2c..855ba0379 100644 --- a/lib/syskit/network_generation/logger.rb +++ b/lib/syskit/network_generation/logger.rb @@ -129,7 +129,7 @@ def self.add_logging_to_network(engine, work_plan) unless (logger_task = deployment.logger_task) warn "deployment #{deployment.process_name} has no usable "\ "logger (default logger name would be "\ - "#{deployment.process_name}_Logger))" + "#{deployment.process_name}_Logger)" next end logger_task = work_plan[deployment.logger_task] From 64d63e4454608ba4f89bfe02249cafecd6f82638 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 28 Nov 2022 17:36:23 -0300 Subject: [PATCH 020/260] feat: handle configuration of the max upload rates --- lib/syskit/roby_app/configuration.rb | 27 ++++++++++++++++++++-- lib/syskit/roby_app/plugin.rb | 3 ++- test/roby_app/test_configuration.rb | 34 ++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index e5e9f2d0e..757ea8701 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -51,7 +51,9 @@ class Configuration LogTransfer = Struct.new( :enabled, :ip, :port, :user, :password, :certificate, - :self_spawned, :target_dir, keyword_init: true + :self_spawned, :target_dir, :default_max_upload_rate, + :max_upload_rates, + keyword_init: true ) do def enabled? enabled @@ -60,6 +62,25 @@ def enabled? def self_spawned? self_spawned end + + # Return the upload rate limit for a given process server + # + # If {#max_upload_rate} contains an entry for this process server + # (keyed by name), it returns it. Otherwise, returns + # {#default_max_upload_rate} + # + # @param [ProcessServerConfig,String] process_server the process server + # object or its name + def max_upload_rate_for(process_server, default: default_max_upload_rate) + name = + if process_server.respond_to?(:name) + process_server.name + else + process_server.to_str + end + + max_upload_rates[name] || default + end end # Configuration of Syskit's log transfer functionality @@ -165,7 +186,9 @@ def initialize(app) password: SecureRandom.base64(32), self_spawned: true, certificate: nil, # Use random generated self-signed certificate - target_dir: nil # Use the app's log dir + target_dir: nil, # Use the app's log dir + default_max_upload_rate: Float::INFINITY, + max_upload_rates: {} ) clear diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index 390c02676..420ec3e66 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -221,7 +221,8 @@ def log_transfer_send_files(name, logfiles) logfiles.each do |path| client.log_upload_file( conf.ip, conf.port, conf.certificate, - conf.user, conf.password, path + conf.user, conf.password, path, + max_upload_rate: conf.max_upload_rate_for(name) ) end client diff --git a/test/roby_app/test_configuration.rb b/test/roby_app/test_configuration.rb index cf017e99f..e806a245a 100644 --- a/test/roby_app/test_configuration.rb +++ b/test/roby_app/test_configuration.rb @@ -155,4 +155,38 @@ def process_server_stop @server_thread = nil end end + + describe "log transfer" do + before do + @conf = Syskit::RobyApp::Configuration.new(Roby.app) + @log_transfer = @conf.log_transfer + end + + describe "#max_upload_rate_for" do + it "returns the default rate if the max_upload_rates hash "\ + "has no entry for the given server" do + default = flexmock + @log_transfer.default_max_upload_rate = default + assert_equal default, @log_transfer.max_upload_rate_for("test") + end + + it "lets the caller set a different default" do + default = flexmock + assert_equal default, + @log_transfer.max_upload_rate_for("test", default: default) + end + + it "finds a process server by name" do + actual = flexmock + @log_transfer.max_upload_rates["test"] = actual + assert_equal actual, @log_transfer.max_upload_rate_for("test") + end + + it "finds a process server by object" do + actual = flexmock + @log_transfer.max_upload_rates["test"] = actual + assert_equal actual, @log_transfer.max_upload_rate_for(flexmock(name: "test")) + end + end + end end From ee6b74ccf9cec3cd6bdb55b8914c0d34780135cc Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Dec 2022 07:19:08 -0300 Subject: [PATCH 021/260] chore: separate most of the log transfer's app integration code from Plugin For better organization and testing --- lib/syskit/roby_app.rb | 4 + lib/syskit/roby_app/configuration.rb | 39 +--- lib/syskit/roby_app/log_transfer_manager.rb | 169 ++++++++++++++++++ .../log_transfer_server/spawn_server.rb | 4 + lib/syskit/roby_app/plugin.rb | 132 +++----------- .../roby_app/remote_processes/server.rb | 55 ++++-- .../remote_processes/test_remote_processes.rb | 34 ++-- test/roby_app/test_log_transfer_manager.rb | 158 ++++++++++++++++ test/roby_app/test_plugin.rb | 100 ----------- 9 files changed, 425 insertions(+), 270 deletions(-) create mode 100644 lib/syskit/roby_app/log_transfer_manager.rb create mode 100644 test/roby_app/test_log_transfer_manager.rb diff --git a/lib/syskit/roby_app.rb b/lib/syskit/roby_app.rb index 7e3e0defe..8f86d0897 100644 --- a/lib/syskit/roby_app.rb +++ b/lib/syskit/roby_app.rb @@ -11,16 +11,20 @@ module RobyApp end end +require "securerandom" + require "syskit/roby_app/logging_configuration" require "syskit/roby_app/logging_group" require "syskit/roby_app/robot_extension" require "syskit/roby_app/toplevel" +require "syskit/roby_app/log_transfer_manager" require "syskit/roby_app/configuration" require "syskit/roby_app/plugin" require "syskit/roby_app/single_file_dsl" require "syskit/roby_app/unmanaged_process" require "syskit/roby_app/unmanaged_tasks_manager" require "syskit/roby_app/remote_processes" +require "syskit/roby_app/tmp_root_ca" require "syskit/roby_app/log_transfer_server" module Syskit diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index 757ea8701..cb65a1aae 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -49,40 +49,6 @@ class Configuration # is 20s attr_accessor :exception_transition_timeout - LogTransfer = Struct.new( - :enabled, :ip, :port, :user, :password, :certificate, - :self_spawned, :target_dir, :default_max_upload_rate, - :max_upload_rates, - keyword_init: true - ) do - def enabled? - enabled - end - - def self_spawned? - self_spawned - end - - # Return the upload rate limit for a given process server - # - # If {#max_upload_rate} contains an entry for this process server - # (keyed by name), it returns it. Otherwise, returns - # {#default_max_upload_rate} - # - # @param [ProcessServerConfig,String] process_server the process server - # object or its name - def max_upload_rate_for(process_server, default: default_max_upload_rate) - name = - if process_server.respond_to?(:name) - process_server.name - else - process_server.to_str - end - - max_upload_rates[name] || default - end - end - # Configuration of Syskit's log transfer functionality # # Minimum configuration: set `ip` to an IP which the process servers @@ -99,7 +65,7 @@ def max_upload_rate_for(process_server, default: default_max_upload_rate) # certificate and set self_spawned to false. In this case, target_dir is # ignored # - # @return [LogTransfer] + # @return [LogTransferManager::Configuration] attr_reader :log_transfer # Period in seconds for triggering log rotation and transfer @@ -179,7 +145,7 @@ def initialize(app) @kill_all_on_process_server_connection = false @log_rotation_period = nil - @log_transfer = LogTransfer.new( + @log_transfer = LogTransferManager::Configuration.new( enabled: false, user: "syskit", port: 22, @@ -192,7 +158,6 @@ def initialize(app) ) clear - self.export_types = true end diff --git a/lib/syskit/roby_app/log_transfer_manager.rb b/lib/syskit/roby_app/log_transfer_manager.rb new file mode 100644 index 000000000..c1e794ecf --- /dev/null +++ b/lib/syskit/roby_app/log_transfer_manager.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +module Syskit + module RobyApp + # High-level interface to log transfer for the benefit of {Plugin} + class LogTransferManager + # @param [Roby::Application] app + # @param [Configuration] conf + def initialize(conf = Syskit.conf.log_transfer) + @conf = conf + + unless @conf.ip + raise ArgumentError, "log transfer is enabled, but the ip is not set" + end + + unless @conf.target_dir + raise ArgumentError, "log transfer is enabled, but target_dir not set" + end + + server_start if @conf.self_spawned? + end + + def dispose(clients, flush: true) + return unless server_started? + + self.flush(clients) if flush + server_stop + end + + # Start an in-process server suitable to receive remote process files + # + # It auto-generates the password and certificates, and updates the + # configuration accordingly + def server_start + raise ArgumentError, "log transfer server already running" if @server + + @self_signed_ca = TmpRootCA.new(@conf.ip) + @conf.user ||= "Syskit" + @conf.password ||= SecureRandom.base64(32) + @conf.certificate = @self_signed_ca.certificate + + @server = LogTransferServer::SpawnServer.new( + @conf.target_dir, @conf.user, @conf.password, + @self_signed_ca.private_certificate_path + ) + @conf.port = @server.port + end + + # Whether files from the given directory should be transferred + def transfer_local_files_from?(dir) + conf.target_dir != dir + end + + # Transfer the given files to the FTP server configured in + # {Configuration#log_transfer} + # + # @param [Array<(String, RemoteProcesses::Client, Array)>] logs + # list of logs to transfer, per remote server + def transfer(logs) + logs.each do |name, client, paths| + transfer_one_process_server_logs(name, client, paths) + end + end + + # @api private + # + # Transfer log files of a single process server + # + # @param [RemoteProcesses::Client] process_server + # @param [Array] logfiles + def transfer_one_process_server_logs(name, client, paths) + paths.each do |path| + client.log_upload_file( + @conf.ip, @conf.port, @conf.certificate, + @conf.user, @conf.password, Pathname(path), + max_upload_rate: @conf.max_upload_rate_for(name) + ) + end + end + + # Whether the in-process transfer server has been started + def server_started? + @server + end + + # Wait for all pending transfers to finish + # + # @return [{RemoteProcesses::Client=>Array}] + def flush(clients, poll_period: 0.5, timeout: 600) + results = {} + deadline = Time.now + timeout + clients.each { |c| results[c] = [] } + loop do + clients = flush_poll_clients(clients, results) + break if clients.empty? + + if Time.now > deadline + raise Timeout::Error, + "failed to flush all pending file transfers "\ + "within #{timeout} seconds" + end + + sleep(poll_period) + end + + results + end + + # @api private + # + # Do a single pass to flush clients + # + # @return the set of clients that are not finished with transfers + def flush_poll_clients(clients, results) + clients.find_all do |c| + state = c.log_upload_state + results[c].concat(state.each_result.to_a) + next(false) if state.pending_count == 0 + + ::Robot.info "Waiting for process server at #{c.host} "\ + "to finish uploading" + true + end + end + + def server_stop + @server.stop + @server.join + @self_signed_ca.dispose + @server = nil + @self_signed_ca = nil + end + + Configuration = Struct.new( + :enabled, :ip, :port, :user, :password, :certificate, + :self_spawned, :target_dir, :default_max_upload_rate, + :max_upload_rates, + keyword_init: true + ) do + def enabled? + enabled + end + + def self_spawned? + self_spawned + end + + # Return the upload rate limit for a given process server + # + # If {#max_upload_rate} contains an entry for this process server + # (keyed by name), it returns it. Otherwise, returns + # {#default_max_upload_rate} + # + # @param [ProcessServerConfig,String] process_server the process server + # object or its name + def max_upload_rate_for(process_server, default: default_max_upload_rate) + name = + if process_server.respond_to?(:name) + process_server.name + else + process_server.to_str + end + + max_upload_rates[name] || default + end + end + end + end +end diff --git a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb index eaa21c483..d88e31797 100644 --- a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb +++ b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb @@ -48,6 +48,10 @@ def run end def stop + dispose + end + + def dispose @server.stop end diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index 420ec3e66..f4ab5887f 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "syskit/roby_app/tmp_root_ca" - class Module def backward_compatible_constant(old_name, new_constant, file) msg = " #{name}::#{old_name} has been renamed to #{new_constant} and is now in #{file}" @@ -35,6 +33,10 @@ module RobyApp # loaded models. In addition, a text notification is sent to inform # a shell user module Plugin + # @return [LogTransferManager,nil] the log transfer support object, or nil + # before {.setup}, or if log transfer is not enabled + attr_accessor :syskit_log_transfer_manager + attr_writer :syskit_use_update_properties # Assume all component models have been migrated to use update_properties @@ -159,101 +161,20 @@ def self.setup(app) rtt_core_model = app.default_loader.task_model_from_name("RTT::TaskContext") Syskit::TaskContext.define_from_orogen(rtt_core_model, register: true) - # Log Transfer FTP Server spawned during Application#setup - log_transfer = Syskit.conf.log_transfer - if log_transfer.enabled? - unless log_transfer.ip - raise ArgumentError, - "log transfer is enabled, but log_transfer.ip is not set" - end - - log_transfer.target_dir ||= Roby.app.log_dir - - unless Syskit.conf.log_rotation_period - raise ArgumentError, - "cannot set up log transfer without log rotation" - end - - app.log_transfer_start_server if log_transfer.self_spawned? - end - end - - def log_transfer_start_server - if @log_transfer_server - raise ArgumentError, "log transfer server already running" - end - - conf = Syskit.conf.log_transfer - @log_transfer_ca = TmpRootCA.new(conf.ip) - conf.certificate = @log_transfer_ca.certificate - - @log_transfer_server = LogTransferServer::SpawnServer.new( - conf.target_dir, conf.user, conf.password, - @log_transfer_ca.private_certificate_path - ) - conf.port = @log_transfer_server.port + log_transfer_setup end - # Transfer the given files to the FTP server configured in - # {Configuration#log_transfer} - # - # @param [{String=>[String]}] logs mapping of server name to the list of - # log names to transfer for this server - def log_transfer_upload(logs) - logs.each do |process_server_name, process_server_logs| - process_server = - Syskit.conf.process_server_config_for(process_server_name) - conf = Syskit.conf.log_transfer - - if process_server.in_process? || process_server.on_localhost? - next if conf.target_dir == Roby.app.log_dir - end + def log_transfer_setup + return unless Syskit.conf.log_transfer.enabled? - log_transfer_send_files(process_server_name, process_server_logs) + unless Syskit.conf.log_rotation_period + raise ArgumentError, + "cannot enable log transfer without log rotation" end - end - - def log_transfer_send_files(name, logfiles) - raise "log transfer server is not started" unless @log_transfer_server - conf = Syskit.conf.log_transfer - client = Syskit.conf.process_server_for(name) - logfiles.each do |path| - client.log_upload_file( - conf.ip, conf.port, conf.certificate, - conf.user, conf.password, path, - max_upload_rate: conf.max_upload_rate_for(name) - ) - end - client - end - - def log_transfer_server_started? - @log_transfer_server - end - - def log_transfer_flush - clients = Syskit.conf.each_process_server.to_a - until clients.empty? - clients = clients.find_all do |c| - state = c.log_upload_state - next(false) if state.pending == 0 - - Robot.info "Waiting for process server at #{c.host} "\ - "to finish uploading" - true - end - - sleep(0.5) unless clients.empty? - end - end - - def log_transfer_stop_server - @log_transfer_server.stop - @log_transfer_server.join - @log_transfer_ca.dispose - @log_transfer_server = nil - @log_transfer_ca = nil + app.syskit_log_transfer_manager = LogTransferManager.new( + Syskit.conf.log_transfer + ) end # Hook called by the main application in Application#setup after @@ -285,13 +206,11 @@ def self.cleanup(app) app.syskit_remove_configuration_changes_listener end + app.syskit_log_transfer_manager&.dispose( + Syskit.conf.each_process_server.to_a + ) disconnect_all_process_servers stop_local_process_server(app) - - if app.log_transfer_server_started? - app.log_transfer_flush - app.log_transfer_stop_server - end end # Hook called by the main application to prepare for execution @@ -300,10 +219,8 @@ def self.prepare(app) if Syskit.conf.log_rotation_period app.execution_engine.every(Syskit.conf.log_rotation_period) do - rotated_logs = app.rotate_logs - if Syskit.conf.log_transfer.enabled? - app.log_transfer_upload(rotated_logs) - end + rotated_logs = app.syskit_rotate_logs + app.log_transfer_manager&.transfer(rotated_logs) end end end @@ -1024,11 +941,14 @@ def self.setup_rest_interface(app, rest_api) rest_api.mount REST_API => "/syskit" end - def rotate_logs - plan.find_tasks(Syskit::LoggerService).running.each_with_object({}) do |task, rotated_logs| - process_server_name = task.log_server_name - (rotated_logs[process_server_name] ||= []).concat(task.rotate_log) - end + def syskit_rotate_logs + plan.find_tasks(Syskit::LoggerService) + .running.each_with_object({}) do |task, rotated_logs| + process_server = Syskit.conf.process_server_config_for( + task.log_server_name + ) + (rotated_logs[process_server] ||= []).concat(task.rotate_log) + end end end end diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 50bdd9003..b500b14cc 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -407,18 +407,10 @@ def handle_command(socket) # :nodoc: elsif cmd_code == COMMAND_QUIT quit elsif cmd_code == COMMAND_LOG_UPLOAD_FILE - host, port, certificate, user, password, localfile, - max_upload_rate = - Marshal.load(socket) - Server.debug "#{socket} requested uploading of #{localfile}" - @log_upload_command_queue << - FTPUpload.new( - host, port, certificate, - user, password, localfile, - max_upload_rate: max_upload_rate || Float::INFINITY - ) - + parameters = Marshal.load(socket) + log_upload_file(socket, parameters) socket.write(RET_YES) + elsif cmd_code == COMMAND_LOG_UPLOAD_STATE state = log_upload_state socket.write RET_YES @@ -575,6 +567,40 @@ def quit @com_w&.write INTERNAL_QUIT end + def log_upload_file(socket, parameters) + host, port, certificate, user, password, localfile, max_upload_rate = + parameters + + Server.debug "#{socket} requested uploading of #{localfile}" + + begin + localfile = log_upload_sanitize_path(Pathname(localfile)) + rescue Exception => e + @log_upload_results_queue << + LogUploadState::Result.new(localfile, false, e.message) + return + end + + Server.info "queueing upload of #{localfile} to #{host}:#{port}" + @log_upload_command_queue << + FTPUpload.new( + host, port, certificate, + user, password, localfile, + max_upload_rate: max_upload_rate || Float::INFINITY + ) + end + + def log_upload_sanitize_path(path) + log_path = Pathname(app.log_dir) + full_path = path.realpath(log_path) + if full_path.to_s.start_with?(log_path.to_s + "/") + return full_path + end + + raise ArgumentError, + "cannot upload files not within the app's log directory" + end + def log_upload_main while (transfer = @log_upload_command_queue.pop) @log_upload_current.set(transfer) @@ -591,6 +617,13 @@ def log_upload_state break end + log_dir = Pathname.new(app.log_dir) + results.each do |r| + if r.success? + r.file = Pathname.new(r.file).relative_path_from(log_dir).to_s + end + end + # This count is not exact. However, it's designed to show # at least one transfer if there are some. I.e. the only # case where pending == 0 is when there is truly nothing to diff --git a/test/roby_app/remote_processes/test_remote_processes.rb b/test/roby_app/remote_processes/test_remote_processes.rb index e8e91de7f..d83b86441 100644 --- a/test/roby_app/remote_processes/test_remote_processes.rb +++ b/test/roby_app/remote_processes/test_remote_processes.rb @@ -19,6 +19,9 @@ Syskit::RobyApp::RemoteProcesses::Server.logger.level = Logger::WARN @__orocos_current_log_level = Orocos.logger.level Orocos.logger.level = Logger::FATAL + + @root_loader = OroGen::Loaders::Aggregate.new + OroGen::Loaders::RTT.setup_loader(root_loader) end after do @@ -39,8 +42,6 @@ describe "#initialize" do it "registers the loader exactly once on the provided root loader" do start_server - root_loader = OroGen::Loaders::Aggregate.new - OroGen::Loaders::RTT.setup_loader(root_loader) client = Syskit::RobyApp::RemoteProcesses::Client.new( "localhost", server.port, @@ -52,7 +53,7 @@ describe "#pid" do before do - start_and_connect_to_server + @client = start_and_connect_to_server end it "returns the process server's PID" do @@ -63,7 +64,7 @@ describe "#loader" do attr_reader :loader before do - start_and_connect_to_server + @client, = start_and_connect_to_server @loader = client.loader end @@ -94,7 +95,7 @@ describe "#start" do before do - start_and_connect_to_server + @client, = start_and_connect_to_server end it "can start a process on the server" do @@ -135,7 +136,7 @@ describe "waits for the process to be running" do before do - start_and_connect_to_server + @client, = start_and_connect_to_server end it "returns a hash with information about a process and its tasks" do @@ -244,7 +245,7 @@ describe "stopping a remote process" do attr_reader :process before do - start_and_connect_to_server + @client, = start_and_connect_to_server @process = client.start( "syskit_tests_empty", "syskit_tests_empty", { "syskit_tests_empty" => "syskit_tests_empty" }, @@ -271,7 +272,7 @@ describe "stopping all remote processes" do before do - start_and_connect_to_server + @client, = start_and_connect_to_server @processes = 10.times.map do |i| client.start( "syskit_tests_empty_#{i}", "syskit_tests_empty", @@ -304,9 +305,10 @@ describe "#log_upload_file" do before do - start_and_connect_to_server + @client, @remote_log_dir = start_and_connect_to_server @port, @certificate = spawn_log_transfer_server - @logfile = File.join(make_tmpdir, "logfile.log") + remote_app_path = Pathname(@remote_log_dir).each_child.first + @logfile = remote_app_path / "logfile.log" File.open(@logfile, "wb") do |f| f.write(SecureRandom.random_bytes(547)) # create random 547 byte end @@ -453,13 +455,13 @@ def start_server end def connect_to_server - @root_loader = OroGen::Loaders::Aggregate.new - OroGen::Loaders::RTT.setup_loader(root_loader) - @client = Syskit::RobyApp::RemoteProcesses::Client.new( - "localhost", - server.port, - root_loader: root_loader + client = Syskit::RobyApp::RemoteProcesses::Client.new( + "localhost", server.port, root_loader: root_loader ) + + log_dir = make_tmpdir + client.create_log_dir(log_dir, Roby.app.time_tag) + [client, log_dir] end def start_and_connect_to_server diff --git a/test/roby_app/test_log_transfer_manager.rb b/test/roby_app/test_log_transfer_manager.rb new file mode 100644 index 000000000..b4b0cba5b --- /dev/null +++ b/test/roby_app/test_log_transfer_manager.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +require "syskit/test/self" +require "syskit/test/roby_app_helpers" +require "syskit/roby_app/remote_processes" + +module Syskit + module RobyApp + describe LogTransferManager do + before do + @server_threads = [] + @clients = [] + @client, @remote_dir_path = create_process_server + + @conf = LogTransferManager::Configuration.new( + ip: "127.0.0.1", + self_spawned: true, + max_upload_rates: {} + ) + @conf.target_dir = make_tmpdir + @manager = nil + end + + after do + @manager&.dispose(@clients) + close_process_servers + end + + it "sets an in-process server and allows file transfers" do + @conf.ip = "127.0.0.1" + file_path = create_test_file(@remote_dir_path) + @manager = LogTransferManager.new(@conf) + assert @manager.server_started? + @manager.transfer([["test", @client, [file_path.basename]]]) + assert_upload_succeeds(file_path, @client) + end + + it "stops the in-process server on dispose" do + @conf.ip = "127.0.0.1" + @manager = LogTransferManager.new(@conf) + # Check that the server is reachable + TCPSocket.new(@conf.ip, @conf.port).close + @manager.dispose([@client]) + assert_raises(Errno::ECONNREFUSED) do + TCPSocket.new(@conf.ip, @conf.port) + end + end + + it "refuses to upload a file that is outside the log dir" do + @conf.ip = "127.0.0.1" + @manager = LogTransferManager.new(@conf) + other_path = make_tmppath + passwd_abs = other_path / "passwd" + passwd_rel = passwd_abs.relative_path_from(@remote_dir_path).to_s + passwd_abs.write("test") + @manager.transfer([["test", @client, [passwd_rel]]]) + assert_upload_fails( + @client, /cannot upload files not within the app's log directory/ + ) + @manager.transfer([["test", @client, [passwd_abs]]]) + assert_upload_fails( + @client, /cannot upload files not within the app's log directory/ + ) + end + + it "handles an externally started server" do + @conf.ip = "127.0.0.1" + @conf.self_spawned = false + @conf.user = "user" + @conf.password = "password" + target_path = make_tmppath + @conf.target_dir = target_path.to_s + ca = TmpRootCA.new("127.0.0.1") + @conf.certificate = ca.certificate + server = LogTransferServer::SpawnServer.new( + target_path.to_s, "user", "password", + ca.private_certificate_path + ) + @conf.port = server.port + + file_path = create_test_file(@remote_dir_path) + @manager = LogTransferManager.new(@conf) + @manager.transfer([["test", @client, [file_path.basename]]]) + assert_upload_succeeds(file_path, @client) + @manager.dispose([@client]) + ensure + server&.dispose + end + + # Spawn a process server + # + # @return [(RemoteProcesses::Client,Pathname)] + def create_process_server + server = RemoteProcesses::Server.new(app, port: 0) + server.open + thread = Thread.new { server.listen } + + client = RemoteProcesses::Client.new("localhost", server.port) + @server_threads << thread + @clients << client + log_dir = config_log_dir(client) + [client, log_dir] + rescue StandardError + server.quit_and_join + raise + end + + def config_log_dir(client) + dir = make_tmppath + client.create_log_dir( + dir.to_s, Roby.app.time_tag, + { "parent" => Roby.app.app_metadata } + ) + dir.each_child.first + end + + def close_process_servers + @clients.each do |client| + client.quit_server + client.close + end + + @server_threads.each(&:join) + end + + def create_test_file(dir_path) + file_path = dir_path / "testfile.log" + file_path.write(SecureRandom.random_bytes(547)) + file_path + end + + def assert_upload_fails(client, match) + result = assert_upload_with_single_result(@manager, client) + + refute result.success? + assert_match(match, result.message) + end + + def assert_upload_succeeds(test_file_path, client) + result = assert_upload_with_single_result(@manager, client) + + assert result.success?, "transfer failed, message: #{result.message}" + assert_equal test_file_path.basename.to_s, result.file + actual_content = File.read( + File.join(@conf.target_dir, result.file) + ) + assert_equal test_file_path.read, actual_content + end + + def assert_upload_with_single_result(manager, client) + transfers = manager.flush([client], timeout: 1) + assert_equal [client], transfers.keys + assert_equal 1, transfers[client].size + transfers[client].first + end + end + end +end diff --git a/test/roby_app/test_plugin.rb b/test/roby_app/test_plugin.rb index e4d70b847..1ea1be2b4 100644 --- a/test/roby_app/test_plugin.rb +++ b/test/roby_app/test_plugin.rb @@ -213,106 +213,6 @@ def perform_app_assertion(result) end end - describe "Local Log Transfer Server" do - before do - @server_threads = [] - @process_servers = [] - register_process_server("test_ps") - app.log_transfer_ip = "127.0.0.1" - end - - after do - close_process_servers - app.log_transfer_ip = nil - end - - it "raises if tries to spawn server when there is already a server running" do - app.setup_local_log_transfer_server - assert_raises(RuntimeError) { app.setup_local_log_transfer_server } - end - - it "uploads file from Process Server" do - app.setup_local_log_transfer_server - ps_logfile = create_test_file(@ps_log_dir) - path = File.join(app.log_dir, "logfile.log") - refute File.exist?(path) - client = app.send_file_transfer_command("test_ps", ps_logfile) - assert_upload_succeeds(client) - assert_equal File.read(path), File.read(ps_logfile) - end - - it "raises if tries to upload when there is no server running" do - ps_logfile = create_test_file(@ps_log_dir) - path = File.join(app.log_dir, "logfile.log") - refute File.exist?(path) - assert_raises(RuntimeError) { app.send_file_transfer_command("test_ps", ps_logfile) } - end - - def register_process_server(name) - server = RemoteProcesses::Server.new(app, port: 0) - server.open - - thread = Thread.new { server.listen } - @server_threads << thread - - client = Syskit.conf.connect_to_orocos_process_server( - name, "localhost", - port: server.port, - model_only_server: false - ) - @process_servers << [name, client] - - config_log_dir(client) - end - - def config_log_dir(client) - @ps_log_dir = make_tmpdir - client.create_log_dir( - @ps_log_dir, Roby.app.time_tag, - { "parent" => Roby.app.app_metadata } - ) - end - - def close_process_servers - @process_servers.each do |name, client| - client.close - Syskit.conf.remove_process_server(name) - end - @server_threads.each do |thread| - thread.raise Interrupt - thread.join - end - end - - def create_test_file(ps_log_dir) - logfile = File.join(ps_log_dir, "logfile.log") - File.open(logfile, "wb") do |f| - f.write(SecureRandom.random_bytes(547)) - end - logfile - end - - def wait_for_upload_completion(client, poll_period: 0.01, timeout: 1) - deadline = Time.now + timeout - loop do - if Time.now > deadline - flunk("timed out while waiting for upload completion") - end - - state = client.log_upload_state - return state if state.pending_count == 0 - - sleep poll_period - end - end - - def assert_upload_succeeds(client) - wait_for_upload_completion(client).each_result do |r| - flunk("upload failed: #{r.message}") unless r.success? - end - end - end - describe "Log Rotation" do before do task_m = Syskit::TaskContext.new_submodel From 2009ade0e8cc5a4ff52410e0ebbfd7c5e8899a88 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Dec 2022 09:40:11 -0300 Subject: [PATCH 022/260] fix: ack the result of remote log directory creation command --- lib/syskit/roby_app/remote_processes/client.rb | 1 + lib/syskit/roby_app/remote_processes/server.rb | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/syskit/roby_app/remote_processes/client.rb b/lib/syskit/roby_app/remote_processes/client.rb index 31da6cdb6..dc7e7614b 100644 --- a/lib/syskit/roby_app/remote_processes/client.rb +++ b/lib/syskit/roby_app/remote_processes/client.rb @@ -220,6 +220,7 @@ def start(process_name, deployment, name_mappings = {}, options = {}) def create_log_dir(log_dir, time_tag, metadata = {}) socket.write(COMMAND_CREATE_LOG) Marshal.dump([log_dir, time_tag, metadata], socket) + wait_for_ack end def queue_death_announcement diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index b500b14cc..405ea26b7 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -343,14 +343,14 @@ def handle_command(socket) # :nodoc: metadata ||= {} # compatible with older clients log_dir = File.expand_path(log_dir) if log_dir create_log_dir(log_dir, time_tag, metadata) - rescue Interrupt - raise - rescue Exception => e + socket.write(RET_YES) + rescue StandardError => e Server.warn "failed to create log directory #{log_dir}: "\ "#{e.message}" (e.backtrace || []).each do |line| Server.warn " #{line}" end + socket.write(RET_NO) end elsif cmd_code == COMMAND_START From bb3dc7df7e83798945a56177852ca0b642b41cbd Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Dec 2022 09:40:40 -0300 Subject: [PATCH 023/260] chore: move log result struct under LogUploadState --- lib/syskit/roby_app/remote_processes/ftp_upload.rb | 12 +++--------- .../roby_app/remote_processes/log_upload_state.rb | 6 ++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/syskit/roby_app/remote_processes/ftp_upload.rb b/lib/syskit/roby_app/remote_processes/ftp_upload.rb index 66023ae39..b75422660 100644 --- a/lib/syskit/roby_app/remote_processes/ftp_upload.rb +++ b/lib/syskit/roby_app/remote_processes/ftp_upload.rb @@ -20,12 +20,6 @@ def initialize( # rubocop:disable Metrics/ParameterLists @max_upload_rate = Float(max_upload_rate) end - Result = Struct.new :file, :success, :message do - def success? - success - end - end - # Create a temporary file with the FTP server's public key, to pass # to FTP.open # @@ -58,12 +52,12 @@ def open # Open the connection and transfer the file # - # @return [UploadResult] + # @return [LogUploadState::Result] def open_and_transfer open { |ftp| transfer(ftp) } - Result.new(@file, true, nil) + LogUploadState::Result.new(@file, true, nil) rescue StandardError => e - Result.new(@file, false, e.message) + LogUploadState::Result.new(@file, false, e.message) end # Do transfer the file through the given connection diff --git a/lib/syskit/roby_app/remote_processes/log_upload_state.rb b/lib/syskit/roby_app/remote_processes/log_upload_state.rb index da659b2f0..e3f2df34b 100644 --- a/lib/syskit/roby_app/remote_processes/log_upload_state.rb +++ b/lib/syskit/roby_app/remote_processes/log_upload_state.rb @@ -7,6 +7,12 @@ module RemoteProcesses class LogUploadState attr_reader :pending_count + Result = Struct.new :file, :success, :message do + def success? + success + end + end + def initialize(pending_count, results) @pending_count = pending_count @results = results From ac77ee3bfe5befcd23fa02ca024c00e5280de5a3 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Dec 2022 12:14:43 -0300 Subject: [PATCH 024/260] fix: only use ProcessServerConfig to manipulate process servers in log transfer code --- lib/syskit/roby_app/log_transfer_manager.rb | 30 +++++++------ test/roby_app/test_log_transfer_manager.rb | 47 +++++++++++---------- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/lib/syskit/roby_app/log_transfer_manager.rb b/lib/syskit/roby_app/log_transfer_manager.rb index c1e794ecf..0c7eaf606 100644 --- a/lib/syskit/roby_app/log_transfer_manager.rb +++ b/lib/syskit/roby_app/log_transfer_manager.rb @@ -20,10 +20,11 @@ def initialize(conf = Syskit.conf.log_transfer) server_start if @conf.self_spawned? end - def dispose(clients, flush: true) + # Stop log transfer + def dispose(process_servers, flush: true) return unless server_started? - self.flush(clients) if flush + self.flush(process_servers) if flush server_stop end @@ -48,7 +49,7 @@ def server_start # Whether files from the given directory should be transferred def transfer_local_files_from?(dir) - conf.target_dir != dir + @conf.target_dir != dir end # Transfer the given files to the FTP server configured in @@ -57,8 +58,8 @@ def transfer_local_files_from?(dir) # @param [Array<(String, RemoteProcesses::Client, Array)>] logs # list of logs to transfer, per remote server def transfer(logs) - logs.each do |name, client, paths| - transfer_one_process_server_logs(name, client, paths) + logs.each do |name, process_server, paths| + transfer_one_process_server_logs(name, process_server, paths) end end @@ -68,9 +69,9 @@ def transfer(logs) # # @param [RemoteProcesses::Client] process_server # @param [Array] logfiles - def transfer_one_process_server_logs(name, client, paths) + def transfer_one_process_server_logs(name, process_server, paths) paths.each do |path| - client.log_upload_file( + process_server.client.log_upload_file( @conf.ip, @conf.port, @conf.certificate, @conf.user, @conf.password, Pathname(path), max_upload_rate: @conf.max_upload_rate_for(name) @@ -86,13 +87,13 @@ def server_started? # Wait for all pending transfers to finish # # @return [{RemoteProcesses::Client=>Array}] - def flush(clients, poll_period: 0.5, timeout: 600) + def flush(process_servers, poll_period: 0.5, timeout: 600) results = {} deadline = Time.now + timeout - clients.each { |c| results[c] = [] } + process_servers.each { |c| results[c] = [] } loop do - clients = flush_poll_clients(clients, results) - break if clients.empty? + process_servers = flush_poll_servers(process_servers, results) + break if process_servers.empty? if Time.now > deadline raise Timeout::Error, @@ -111,10 +112,11 @@ def flush(clients, poll_period: 0.5, timeout: 600) # Do a single pass to flush clients # # @return the set of clients that are not finished with transfers - def flush_poll_clients(clients, results) - clients.find_all do |c| + def flush_poll_servers(process_servers, results) + process_servers.find_all do |config| + c = config.client state = c.log_upload_state - results[c].concat(state.each_result.to_a) + results[config].concat(state.each_result.to_a) next(false) if state.pending_count == 0 ::Robot.info "Waiting for process server at #{c.host} "\ diff --git a/test/roby_app/test_log_transfer_manager.rb b/test/roby_app/test_log_transfer_manager.rb index b4b0cba5b..9d7b00dc2 100644 --- a/test/roby_app/test_log_transfer_manager.rb +++ b/test/roby_app/test_log_transfer_manager.rb @@ -9,8 +9,8 @@ module RobyApp describe LogTransferManager do before do @server_threads = [] - @clients = [] - @client, @remote_dir_path = create_process_server + @process_servers = [] + @process_server = create_process_server @conf = LogTransferManager::Configuration.new( ip: "127.0.0.1", @@ -22,17 +22,17 @@ module RobyApp end after do - @manager&.dispose(@clients) + @manager&.dispose(@process_servers) close_process_servers end it "sets an in-process server and allows file transfers" do @conf.ip = "127.0.0.1" - file_path = create_test_file(@remote_dir_path) + file_path = create_test_file(@process_server.log_dir) @manager = LogTransferManager.new(@conf) assert @manager.server_started? - @manager.transfer([["test", @client, [file_path.basename]]]) - assert_upload_succeeds(file_path, @client) + @manager.transfer([["test", @process_server, [file_path.basename]]]) + assert_upload_succeeds(file_path, @process_server) end it "stops the in-process server on dispose" do @@ -40,7 +40,7 @@ module RobyApp @manager = LogTransferManager.new(@conf) # Check that the server is reachable TCPSocket.new(@conf.ip, @conf.port).close - @manager.dispose([@client]) + @manager.dispose([@process_server]) assert_raises(Errno::ECONNREFUSED) do TCPSocket.new(@conf.ip, @conf.port) end @@ -51,15 +51,17 @@ module RobyApp @manager = LogTransferManager.new(@conf) other_path = make_tmppath passwd_abs = other_path / "passwd" - passwd_rel = passwd_abs.relative_path_from(@remote_dir_path).to_s + passwd_rel = passwd_abs.relative_path_from(@process_server.log_dir).to_s passwd_abs.write("test") - @manager.transfer([["test", @client, [passwd_rel]]]) + @manager.transfer([["test", @process_server, [passwd_rel]]]) assert_upload_fails( - @client, /cannot upload files not within the app's log directory/ + @process_server, + /cannot upload files not within the app's log directory/ ) - @manager.transfer([["test", @client, [passwd_abs]]]) + @manager.transfer([["test", @process_server, [passwd_abs]]]) assert_upload_fails( - @client, /cannot upload files not within the app's log directory/ + @process_server, + /cannot upload files not within the app's log directory/ ) end @@ -78,11 +80,11 @@ module RobyApp ) @conf.port = server.port - file_path = create_test_file(@remote_dir_path) + file_path = create_test_file(@process_server.log_dir) @manager = LogTransferManager.new(@conf) - @manager.transfer([["test", @client, [file_path.basename]]]) - assert_upload_succeeds(file_path, @client) - @manager.dispose([@client]) + @manager.transfer([["test", @process_server, [file_path.basename]]]) + assert_upload_succeeds(file_path, @process_server) + @manager.dispose([@process_server]) ensure server&.dispose end @@ -96,10 +98,11 @@ def create_process_server thread = Thread.new { server.listen } client = RemoteProcesses::Client.new("localhost", server.port) - @server_threads << thread - @clients << client log_dir = config_log_dir(client) - [client, log_dir] + config = Configuration::ProcessServerConfig.new("test", client, log_dir) + @server_threads << thread + @process_servers << config + config rescue StandardError server.quit_and_join raise @@ -115,9 +118,9 @@ def config_log_dir(client) end def close_process_servers - @clients.each do |client| - client.quit_server - client.close + @process_servers.each do |c| + c.client.quit_server + c.client.close end @server_threads.each(&:join) From 48f01a8bc5318f1929a34334f1bb1c8926033588 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Dec 2022 16:06:16 -0300 Subject: [PATCH 025/260] fix: log transfer integration within Plugin --- lib/syskit/roby_app/configuration.rb | 11 ++++- lib/syskit/roby_app/plugin.rb | 49 ++++++++++++++++----- test/roby_app/test_configuration.rb | 10 +++++ test/roby_app/test_plugin.rb | 65 +++++++++++++++++++++++++--- 4 files changed, 116 insertions(+), 19 deletions(-) diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index cb65a1aae..0d59793ea 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -596,11 +596,14 @@ def connect_to_orocos_process_server( { "parent" => Roby.app.app_metadata } ) client.kill_all if kill_all_on_process_server_connection? - register_process_server(name, client, log_dir, host_id: host_id || name) + config = register_process_server( + name, client, log_dir, host_id: host_id || name + ) + config.supports_log_transfer = true client end - ProcessServerConfig = Struct.new :name, :client, :log_dir, :host_id do + ProcessServerConfig = Struct.new :name, :client, :log_dir, :host_id, :supports_log_transfer do def on_localhost? host_id == "localhost" || host_id == "syskit" end @@ -612,6 +615,10 @@ def in_process? def loader client.loader end + + def supports_log_transfer? + supports_log_transfer + end end # Make a process server available to syskit diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index f4ab5887f..8d85e98a2 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -161,20 +161,50 @@ def self.setup(app) rtt_core_model = app.default_loader.task_model_from_name("RTT::TaskContext") Syskit::TaskContext.define_from_orogen(rtt_core_model, register: true) - log_transfer_setup + app.syskit_log_transfer_setup end - def log_transfer_setup - return unless Syskit.conf.log_transfer.enabled? + def syskit_log_transfer_setup + return unless Syskit.conf.log_transfer.ip unless Syskit.conf.log_rotation_period raise ArgumentError, "cannot enable log transfer without log rotation" end - app.syskit_log_transfer_manager = LogTransferManager.new( - Syskit.conf.log_transfer - ) + @syskit_log_transfer_manager = + LogTransferManager.new(Syskit.conf.log_transfer) + end + + def syskit_log_transfer_cleanup + syskit_log_transfer_manager&.dispose(syskit_log_transfer_process_servers) + @syskit_log_transfer_manager = nil + end + + # Returns the list of remote process servers that will take part in the + # log transfer process + # + # @return [Configuration::ProcessServerConfig] + def syskit_log_transfer_process_servers + Syskit.conf.each_process_server_config.find_all do |config| + next unless config.supports_log_transfer? + next(true) unless config.on_localhost? + + syskit_log_transfer_manager&.transfer_local_files_from?(config.log_dir) + end + end + + # Rotate logs, and transfer the old logs if log transfer is configured + def syskit_log_perform_rotation_and_transfer + rotated_logs = syskit_rotate_logs + + return unless (mng = syskit_log_transfer_manager) + + handled = syskit_log_transfer_process_servers + rotated_logs.delete_if do |process_server_config, _| + !handled.include?(process_server_config) + end + mng.transfer(rotated_logs) end # Hook called by the main application in Application#setup after @@ -206,9 +236,7 @@ def self.cleanup(app) app.syskit_remove_configuration_changes_listener end - app.syskit_log_transfer_manager&.dispose( - Syskit.conf.each_process_server.to_a - ) + app.syskit_log_transfer_cleanup disconnect_all_process_servers stop_local_process_server(app) end @@ -219,8 +247,7 @@ def self.prepare(app) if Syskit.conf.log_rotation_period app.execution_engine.every(Syskit.conf.log_rotation_period) do - rotated_logs = app.syskit_rotate_logs - app.log_transfer_manager&.transfer(rotated_logs) + app.syskit_log_perform_rotation_and_transfer end end end diff --git a/test/roby_app/test_configuration.rb b/test/roby_app/test_configuration.rb index e806a245a..b87620302 100644 --- a/test/roby_app/test_configuration.rb +++ b/test/roby_app/test_configuration.rb @@ -117,6 +117,16 @@ def stub_deployment(name) process = client.start "deployment", deployment_m.orogen_model assert_equal 20, process.pid end + + it "marks the process server as supporting log transfer" do + process_server_start + @conf.connect_to_orocos_process_server( + "test-remote", "localhost", + port: process_server_port + ) + config = @conf.process_server_config_for("test-remote") + assert config.supports_log_transfer? + end end def process_server_create diff --git a/test/roby_app/test_plugin.rb b/test/roby_app/test_plugin.rb index 1ea1be2b4..29d46ce90 100644 --- a/test/roby_app/test_plugin.rb +++ b/test/roby_app/test_plugin.rb @@ -213,12 +213,24 @@ def perform_app_assertion(result) end end - describe "Log Rotation" do + describe "log rotation and transfer" do before do + Syskit.conf.log_rotation_period = 600 + Syskit.conf.log_transfer.ip = "127.0.0.1" + Syskit.conf.log_transfer.self_spawned = false + Syskit.conf.log_transfer.target_dir = make_tmpdir + end + + after do + Syskit.conf.log_rotation_period = nil + Syskit.conf.log_transfer.ip = nil + app.syskit_log_transfer_cleanup + end + + it "rotates logs and returns which logs were rotated" do task_m = Syskit::TaskContext.new_submodel + task_m.provides Syskit::LoggerService task_m.class_eval do - provides Syskit::LoggerService - def log_server_name "stubs" end @@ -227,12 +239,53 @@ def rotate_log ["old_log_file.log"] end end + @task = syskit_stub_deploy_configure_and_start(task_m) + rotated_logs = app.syskit_rotate_logs + + stubs = Syskit.conf.process_server_config_for("stubs") + assert_equal({ stubs => ["old_log_file.log"] }, rotated_logs) end - it "rotates log" do - rotated_logs = app.rotate_logs - assert_equal({ "stubs" => ["old_log_file.log"] }, rotated_logs) + it "returns an empty list of process servers "\ + "if log transfer is disabled" do + conf = Syskit.conf.process_server_config_for("localhost") + flexmock(conf).should_receive(supports_log_transfer?: true) + assert_equal [], app.syskit_log_transfer_process_servers + end + + it "returns the list of process servers whose logs we want to transfer" do + app.syskit_log_transfer_setup + + conf = Syskit.conf.process_server_config_for("localhost") + flexmock(conf).should_receive(supports_log_transfer?: true) + assert_equal [conf], app.syskit_log_transfer_process_servers + end + + it "ignores local process servers if they have the same directory than "\ + "the transfer's target dir" do + Syskit.conf.log_transfer.target_dir = app.log_dir + app.syskit_log_transfer_setup + + conf = Syskit.conf.process_server_config_for("localhost") + flexmock(conf).should_receive(supports_log_transfer?: true) + assert_equal [], app.syskit_log_transfer_process_servers + end + + it "transfers data for the selected process servers" do + conf = Syskit.conf.process_server_config_for("localhost") + flexmock(conf).should_receive(supports_log_transfer?: true) + flexmock(app) + .should_receive(:syskit_rotate_logs) + .and_return( + { conf => ["old_log_file.log"], + Configuration::ProcessServerConfig.new => ["some_file"] } + ) + + app.syskit_log_transfer_setup + flexmock(app.syskit_log_transfer_manager) + .should_receive(:transfer).with({ conf => ["old_log_file.log"] }) + app.syskit_log_perform_rotation_and_transfer end end From 8f78f27a3bc2958aceeec956b693f77efb296a3c Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Dec 2022 16:15:39 -0300 Subject: [PATCH 026/260] fix: make loggers per-server, and inhibit log messages in tests --- lib/syskit/roby_app/log_transfer_manager.rb | 2 +- .../roby_app/remote_processes/server.rb | 91 ++++++++++--------- test/roby_app/test_log_transfer_manager.rb | 1 + 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/lib/syskit/roby_app/log_transfer_manager.rb b/lib/syskit/roby_app/log_transfer_manager.rb index 0c7eaf606..b7c378f79 100644 --- a/lib/syskit/roby_app/log_transfer_manager.rb +++ b/lib/syskit/roby_app/log_transfer_manager.rb @@ -120,7 +120,7 @@ def flush_poll_servers(process_servers, results) next(false) if state.pending_count == 0 ::Robot.info "Waiting for process server at #{c.host} "\ - "to finish uploading" + "to finish uploading" true end end diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 405ea26b7..993a2243f 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -23,6 +23,9 @@ class Server "Syskit::RobyApp::RemoteProcesses::Server", Logger::INFO ) + include Logger::Forward + include Logger::Hierarchy + # Returns a unique directory name as a subdirectory of # +base_dir+, based on +path_spec+. The generated name # is of the form @@ -148,7 +151,7 @@ def exec INTERNAL_SIGCHLD_TRIGGERED = "S" def open(fd: nil) - Server.info "starting on port #{required_port}" + info "starting on port #{required_port}" server = if fd @@ -179,7 +182,7 @@ def close # # All started processes are stopped when the server quits def listen - Server.info "process server listening on port #{port}" + info "process server listening on port #{port}" server_io, com_r = *@all_ios[0, 2] @quit = false @@ -192,7 +195,7 @@ def listen Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true ) client_socket.fcntl(Fcntl::FD_CLOEXEC, 1) - Server.debug "new connection: #{client_socket}" + debug "new connection: #{client_socket}" @all_ios << client_socket end @@ -205,28 +208,27 @@ def listen elsif cmd == INTERNAL_QUIT next elsif cmd - Server.warn "unknown internal communication code "\ - "#{cmd.inspect}" + warn "unknown internal communication code #{cmd.inspect}" end end readable_sockets.each do |socket| unless handle_command(socket) - Server.debug "#{socket} closed or errored" + debug "#{socket} closed or errored" socket.close @all_ios.delete(socket) end end end - Server.info "process server exited normally" + info "process server exited normally" rescue Interrupt - Server.warn "process server exited after SIGINT" + warn "process server exited after SIGINT" rescue Exception => e - Server.fatal "process server exited because of unhandled exception" - Server.fatal "#{e.message} #{e.class}" + fatal "process server exited because of unhandled exception" + fatal "#{e.message} #{e.class}" e.backtrace.each do |line| - Server.fatal " #{line}" + fatal " #{line}" end ensure quit_and_join @@ -291,13 +293,13 @@ def handle_dead_subprocess(exit_pid, exit_status) # terminated processes def announce_dead_processes(dead_processes) dead_processes.each do |process, exit_status| - Server.debug "announcing death of #{process.name}" + debug "announcing death of #{process.name}" each_client do |socket| - Server.debug " announcing to #{socket}" + debug " announcing to #{socket}" socket.write(EVENT_DEAD_PROCESS) Marshal.dump([process.name, exit_status], socket) rescue SystemCallError, IOError => e - Server.debug " #{socket}: #{e}" + debug " #{socket}: #{e}" end end end @@ -306,9 +308,9 @@ def announce_dead_processes(dead_processes) def quit_and_join # :nodoc: @log_upload_command_queue << nil - Server.info "stopping process server" + info "stopping process server" processes.each_value do |p| - Server.info "killing #{p.name}" + info "killing #{p.name}" # Kill the process hard. If there are still processes, # it means that the normal cleanup procedure did not # work. Not the time to call stop or whatnot @@ -329,14 +331,14 @@ def handle_command(socket) # :nodoc: return false unless cmd_code if cmd_code == COMMAND_GET_PID - Server.debug "#{socket} requested PID" + debug "#{socket} requested PID" Marshal.dump([::Process.pid], socket) elsif cmd_code == COMMAND_GET_INFO - Server.debug "#{socket} requested system information" + debug "#{socket} requested system information" Marshal.dump(build_system_info, socket) elsif cmd_code == COMMAND_CREATE_LOG - Server.debug "#{socket} requested creating a log directory" + debug "#{socket} requested creating a log directory" log_dir, time_tag, metadata = Marshal.load(socket) begin @@ -345,10 +347,10 @@ def handle_command(socket) # :nodoc: create_log_dir(log_dir, time_tag, metadata) socket.write(RET_YES) rescue StandardError => e - Server.warn "failed to create log directory #{log_dir}: "\ - "#{e.message}" + warn "failed to create log directory #{log_dir}: "\ + "#{e.message}" (e.backtrace || []).each do |line| - Server.warn " #{line}" + warn " #{line}" end socket.write(RET_NO) end @@ -357,29 +359,29 @@ def handle_command(socket) # :nodoc: name, deployment_name, name_mappings, options = Marshal.load(socket) options ||= {} - Server.debug "#{socket} requested startup of #{name} with "\ - "#{options} and mappings #{name_mappings}" + debug "#{socket} requested startup of #{name} with "\ + "#{options} and mappings #{name_mappings}" begin p = start_process( name, deployment_name, name_mappings, options ) - Server.debug "#{name}, from #{deployment_name}, "\ - "is started (#{p.pid})" + debug "#{name}, from #{deployment_name}, "\ + "is started (#{p.pid})" socket.write(RET_STARTED_PROCESS) Marshal.dump(p.pid, socket) rescue Interrupt raise rescue Exception => e - Server.warn "failed to start #{name}: #{e.message}" + warn "failed to start #{name}: #{e.message}" (e.backtrace || []).each do |line| - Server.warn " #{line}" + warn " #{line}" end socket.write(RET_NO) socket.write Marshal.dump(e.message) end elsif cmd_code == COMMAND_END name, cleanup, hard = Marshal.load(socket) - Server.debug "#{socket} requested end of #{name}" + debug "#{socket} requested end of #{name}" if (p = processes[name]) begin end_process(p, cleanup: cleanup, hard: hard) @@ -387,18 +389,17 @@ def handle_command(socket) # :nodoc: rescue Interrupt raise rescue Exception => e - Server.warn "exception raised while calling "\ - "#{p}#kill(false)" - Server.log_pp(:warn, e) + warn "exception raised while calling #{p}#kill(false)" + log_pp(:warn, e) socket.write(RET_NO) end else - Server.warn "no process named #{name} to end" + warn "no process named #{name} to end" socket.write(RET_NO) end elsif cmd_code == COMMAND_KILL_ALL cleanup, hard = Marshal.load(socket) - Server.debug "#{socket} requested the end of all processes" + debug "#{socket} requested the end of all processes" processes = kill_all(cleanup: cleanup, hard: hard) dead = join_all(processes) socket.write(RET_YES) @@ -424,14 +425,14 @@ def handle_command(socket) # :nodoc: iors = p.wait_running(0) result[p_name] = ({ iors: iors } if iors) rescue Orocos::NotFound => e - Server.warn(e.message) + warn(e.message) result[p_name] = { error: e.message } rescue Orocos::InvalidIORMessage => e - Server.warn(e.message) + warn(e.message) result[p_name] = { error: e.message } end else - Server.warn("no process named #{p_name} to wait running") + warn("no process named #{p_name} to wait running") result[p_name] = { error: "no process named #{p_name} to wait running" } @@ -451,10 +452,10 @@ def handle_command(socket) # :nodoc: rescue EOFError false rescue Exception => e - Server.fatal "protocol error on #{socket}: #{e}" - Server.fatal "while serving command #{cmd_code}" + fatal "protocol error on #{socket}: #{e}" + fatal "while serving command #{cmd_code}" e.backtrace.each do |bt| - Server.fatal " #{bt}" + fatal " #{bt}" end false end @@ -474,12 +475,12 @@ def create_log_dir(log_dir, time_tag, metadata = {}) app.add_app_metadata(metadata) app.find_and_create_log_dir(time_tag) if (parent_info = metadata["parent"]) - ::Robot.info "created #{app.log_dir} on behalf of" + info "created #{app.log_dir} on behalf of" YAML.dump(parent_info).each_line do |line| - ::Robot.info " #{line.chomp}" + info " #{line.chomp}" end else - ::Robot.info "created #{app.log_dir}" + info "created #{app.log_dir}" end end @@ -571,7 +572,7 @@ def log_upload_file(socket, parameters) host, port, certificate, user, password, localfile, max_upload_rate = parameters - Server.debug "#{socket} requested uploading of #{localfile}" + debug "#{socket} requested uploading of #{localfile}" begin localfile = log_upload_sanitize_path(Pathname(localfile)) @@ -581,7 +582,7 @@ def log_upload_file(socket, parameters) return end - Server.info "queueing upload of #{localfile} to #{host}:#{port}" + info "queueing upload of #{localfile} to #{host}:#{port}" @log_upload_command_queue << FTPUpload.new( host, port, certificate, diff --git a/test/roby_app/test_log_transfer_manager.rb b/test/roby_app/test_log_transfer_manager.rb index 9d7b00dc2..032654222 100644 --- a/test/roby_app/test_log_transfer_manager.rb +++ b/test/roby_app/test_log_transfer_manager.rb @@ -94,6 +94,7 @@ module RobyApp # @return [(RemoteProcesses::Client,Pathname)] def create_process_server server = RemoteProcesses::Server.new(app, port: 0) + server.make_own_logger("", Logger::FATAL) server.open thread = Thread.new { server.listen } From 04936e426cd487463ab79b70b8a326e1d19e5bc4 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Dec 2022 16:20:43 -0300 Subject: [PATCH 027/260] fix: inhibit last log messages in test_log_transfer_manager --- test/roby_app/test_log_transfer_manager.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/roby_app/test_log_transfer_manager.rb b/test/roby_app/test_log_transfer_manager.rb index 032654222..8915837b5 100644 --- a/test/roby_app/test_log_transfer_manager.rb +++ b/test/roby_app/test_log_transfer_manager.rb @@ -152,7 +152,10 @@ def assert_upload_succeeds(test_file_path, client) end def assert_upload_with_single_result(manager, client) - transfers = manager.flush([client], timeout: 1) + transfers = nil + capture_log(::Robot, :info) do + transfers = manager.flush([client], timeout: 1) + end assert_equal [client], transfers.keys assert_equal 1, transfers[client].size transfers[client].first From 84ef5dcf49fd959cc36a372c7f064ba6550b4b19 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Dec 2022 17:07:26 -0300 Subject: [PATCH 028/260] fix: type mismatch in argument passed to LogTransferManager#transfer --- lib/syskit/roby_app/log_transfer_manager.rb | 9 +++++---- test/roby_app/test_log_transfer_manager.rb | 8 ++++---- test/roby_app/test_plugin.rb | 12 ++++++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/syskit/roby_app/log_transfer_manager.rb b/lib/syskit/roby_app/log_transfer_manager.rb index b7c378f79..77f1a2ef7 100644 --- a/lib/syskit/roby_app/log_transfer_manager.rb +++ b/lib/syskit/roby_app/log_transfer_manager.rb @@ -58,8 +58,8 @@ def transfer_local_files_from?(dir) # @param [Array<(String, RemoteProcesses::Client, Array)>] logs # list of logs to transfer, per remote server def transfer(logs) - logs.each do |name, process_server, paths| - transfer_one_process_server_logs(name, process_server, paths) + logs.each do |process_server, paths| + transfer_one_process_server_logs(process_server, paths) end end @@ -69,12 +69,13 @@ def transfer(logs) # # @param [RemoteProcesses::Client] process_server # @param [Array] logfiles - def transfer_one_process_server_logs(name, process_server, paths) + def transfer_one_process_server_logs(process_server, paths) + upload_rate = @conf.max_upload_rate_for(process_server.name) paths.each do |path| process_server.client.log_upload_file( @conf.ip, @conf.port, @conf.certificate, @conf.user, @conf.password, Pathname(path), - max_upload_rate: @conf.max_upload_rate_for(name) + max_upload_rate: upload_rate ) end end diff --git a/test/roby_app/test_log_transfer_manager.rb b/test/roby_app/test_log_transfer_manager.rb index 8915837b5..af126058f 100644 --- a/test/roby_app/test_log_transfer_manager.rb +++ b/test/roby_app/test_log_transfer_manager.rb @@ -31,7 +31,7 @@ module RobyApp file_path = create_test_file(@process_server.log_dir) @manager = LogTransferManager.new(@conf) assert @manager.server_started? - @manager.transfer([["test", @process_server, [file_path.basename]]]) + @manager.transfer([[@process_server, [file_path.basename]]]) assert_upload_succeeds(file_path, @process_server) end @@ -53,12 +53,12 @@ module RobyApp passwd_abs = other_path / "passwd" passwd_rel = passwd_abs.relative_path_from(@process_server.log_dir).to_s passwd_abs.write("test") - @manager.transfer([["test", @process_server, [passwd_rel]]]) + @manager.transfer([[@process_server, [passwd_rel]]]) assert_upload_fails( @process_server, /cannot upload files not within the app's log directory/ ) - @manager.transfer([["test", @process_server, [passwd_abs]]]) + @manager.transfer([[@process_server, [passwd_abs]]]) assert_upload_fails( @process_server, /cannot upload files not within the app's log directory/ @@ -82,7 +82,7 @@ module RobyApp file_path = create_test_file(@process_server.log_dir) @manager = LogTransferManager.new(@conf) - @manager.transfer([["test", @process_server, [file_path.basename]]]) + @manager.transfer([[@process_server, [file_path.basename]]]) assert_upload_succeeds(file_path, @process_server) @manager.dispose([@process_server]) ensure diff --git a/test/roby_app/test_plugin.rb b/test/roby_app/test_plugin.rb index 29d46ce90..f2a6de755 100644 --- a/test/roby_app/test_plugin.rb +++ b/test/roby_app/test_plugin.rb @@ -273,6 +273,10 @@ def rotate_log end it "transfers data for the selected process servers" do + Syskit.conf.log_transfer.user = "user" + Syskit.conf.log_transfer.password = "pass" + Syskit.conf.log_transfer.certificate = "cert" + Syskit.conf.log_transfer.port = 42 conf = Syskit.conf.process_server_config_for("localhost") flexmock(conf).should_receive(supports_log_transfer?: true) flexmock(app) @@ -283,8 +287,12 @@ def rotate_log ) app.syskit_log_transfer_setup - flexmock(app.syskit_log_transfer_manager) - .should_receive(:transfer).with({ conf => ["old_log_file.log"] }) + flexmock(conf.client) + .should_receive(:log_upload_file).explicitly + .with("127.0.0.1", 42, "cert", "user", "pass", + Pathname("old_log_file.log"), + max_upload_rate: Float::INFINITY) + .once app.syskit_log_perform_rotation_and_transfer end end From a27460c020e9211732a4e3f81a97d53b069e4765 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Dec 2022 17:07:40 -0300 Subject: [PATCH 029/260] fix: set supports_log_transfer on the local process server --- lib/syskit/roby_app/plugin.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index 8d85e98a2..3ceec93ff 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -669,7 +669,8 @@ def self.connect_to_local_process_server(app) client = create_local_process_server_client(app) # Do *not* manage the log directory for that one ... - Syskit.conf.register_process_server("localhost", client, app.log_dir) + conf = Syskit.conf.register_process_server("localhost", client, app.log_dir) + conf.supports_log_transfer = true client end From 68e06a32278727ea73eab674324d783197d4ed50 Mon Sep 17 00:00:00 2001 From: caioaamaral Date: Fri, 2 Dec 2022 10:21:42 -0300 Subject: [PATCH 030/260] feat(gui): add checkbox 'Run on single mode' --- lib/syskit/gui/app_start_dialog.rb | 18 +++++++++++++++++- lib/syskit/gui/runtime_state.rb | 3 ++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/syskit/gui/app_start_dialog.rb b/lib/syskit/gui/app_start_dialog.rb index 7ec772fa4..d0b6df1d4 100644 --- a/lib/syskit/gui/app_start_dialog.rb +++ b/lib/syskit/gui/app_start_dialog.rb @@ -15,6 +15,12 @@ class AppStartDialog < Qt::Dialog # @return [Qt::CheckBox] attr_reader :start_controller + # The checkbox allowing to choose whether Roby app should be run + # on single mode. For more information, check [Roby::Application#single] + # + # @return [Qt::CheckBox] + attr_reader :single + # Text used to allow the user to not load any robot configuration NO_ROBOT = " -- None -- " @@ -37,6 +43,9 @@ def initialize(names, parent = nil, default_robot_name: "default") layout.add_widget(@start_controller = Qt::CheckBox.new("Start controller")) start_controller.checked = true + layout.add_widget(@single = Qt::CheckBox.new("Run on single mode")) + single.checked = false + button_box = Qt::DialogButtonBox.new( Qt::DialogButtonBox::Ok | Qt::DialogButtonBox::Cancel ) @@ -64,6 +73,13 @@ def start_controller? start_controller.checked? end + # Whether Roby app should be run on single mode + # + # @return [Boolean] + def single? + single.checked? + end + # Executes a {AppStartDialog} in a modal way and returns the result # # @return [nil,(String,Boolean)] either nil if the dialog was @@ -74,7 +90,7 @@ def start_controller? def self.exec(names, parent = nil, default_robot_name: "default") dialog = new(names, parent, default_robot_name: default_robot_name) if Qt::Dialog::Accepted == dialog.exec - [dialog.selected_name, dialog.start_controller?] + [dialog.selected_name, dialog.start_controller?, dialog.single?] end end end diff --git a/lib/syskit/gui/runtime_state.rb b/lib/syskit/gui/runtime_state.rb index 8144c57cf..e03af281b 100644 --- a/lib/syskit/gui/runtime_state.rb +++ b/lib/syskit/gui/runtime_state.rb @@ -287,7 +287,7 @@ def remote_name end def app_start(robot_name: "default", port: nil) - robot_name, start_controller = AppStartDialog.exec( + robot_name, start_controller, single = AppStartDialog.exec( Roby.app.robots.names, self, default_robot_name: robot_name ) return unless robot_name @@ -296,6 +296,7 @@ def app_start(robot_name: "default", port: nil) extra_args << "-r" << robot_name unless robot_name.empty? extra_args << "-c" if start_controller extra_args << "--port=#{port}" if port + extra_args << "--single" if single extra_args.concat( Roby.app.argv_set.flat_map { |arg| ["--set", arg] } ) From 046864d2764e75212b1985e7c676489f25e25cb1 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 5 Dec 2022 20:44:35 -0300 Subject: [PATCH 031/260] fix: implicit FTPS usage should be default only after 2.7.0 This workarounds some incompatibility between net-ftp and ftpd. They don't manage connecting properly in implicit mode before 2.7.0, and don't manage connecting properly in explicit mode afterwards --- .../roby_app/log_transfer_server/spawn_server.rb | 14 ++++++++++++-- lib/syskit/roby_app/remote_processes/ftp_upload.rb | 2 +- test/roby_app/spawn_server/test_spawn_server.rb | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb index d88e31797..f9f81c542 100644 --- a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb +++ b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb @@ -4,7 +4,17 @@ module Syskit module RobyApp - module LogTransferServer + module LogTransferServer # :nodoc: + # Whether we should configure client and server to use implicit FTPs by + # default + # + # This workarounds some incompatibility between net-ftp and ftpd. They + # don't manage connecting properly in implicit mode before 2.7.0, and + # don't manage connecting properly in explicit mode afterwards + def self.use_implicit_ftps? + RUBY_VERSION >= "2.7.0" + end + # Class responsible for spawning an FTP server for transfering logs class SpawnServer attr_reader :port @@ -16,7 +26,7 @@ def initialize( password, certfile_path, interface: "127.0.0.1", - tls: :implicit, + tls: LogTransferServer.use_implicit_ftps? ? :implicit : :explicit, port: 0, session_timeout: default_session_timeout, nat_ip: nil, diff --git a/lib/syskit/roby_app/remote_processes/ftp_upload.rb b/lib/syskit/roby_app/remote_processes/ftp_upload.rb index b75422660..94cf2afec 100644 --- a/lib/syskit/roby_app/remote_processes/ftp_upload.rb +++ b/lib/syskit/roby_app/remote_processes/ftp_upload.rb @@ -40,7 +40,7 @@ def open Net::FTP.open( @host, private_data_connection: false, port: @port, - implicit_ftps: true, + implicit_ftps: LogTransferServer.use_implicit_ftps?, ssl: { verify_mode: OpenSSL::SSL::VERIFY_PEER, ca_file: cert_path } ) do |ftp| diff --git a/test/roby_app/spawn_server/test_spawn_server.rb b/test/roby_app/spawn_server/test_spawn_server.rb index ad89e784c..5e1eb8533 100644 --- a/test/roby_app/spawn_server/test_spawn_server.rb +++ b/test/roby_app/spawn_server/test_spawn_server.rb @@ -28,7 +28,7 @@ def ftp_open(certfile_path: @certfile_path, &block) Net::FTP.open( "localhost", port: @server.port, - implicit_ftps: true, + implicit_ftps: LogTransferServer.use_implicit_ftps?, ssl: { verify_mode: OpenSSL::SSL::VERIFY_PEER, verify_hostname: false, ca_file: certfile_path }, From e8110fcccdcc2b99541d244749160b26197665e0 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 5 Dec 2022 20:44:42 -0300 Subject: [PATCH 032/260] fix: trivial unit test fix --- test/roby_app/remote_processes/test_remote_processes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/roby_app/remote_processes/test_remote_processes.rb b/test/roby_app/remote_processes/test_remote_processes.rb index d83b86441..0b1f2cce1 100644 --- a/test/roby_app/remote_processes/test_remote_processes.rb +++ b/test/roby_app/remote_processes/test_remote_processes.rb @@ -53,7 +53,7 @@ describe "#pid" do before do - @client = start_and_connect_to_server + @client, = start_and_connect_to_server end it "returns the process server's PID" do From 89129b9b3c3f998d34413355cf6375517ddd3207 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 5 Dec 2022 21:55:35 -0300 Subject: [PATCH 033/260] fix: rubocop offenses --- lib/syskit/roby_app/remote_processes/server.rb | 4 +--- test/roby_app/spawn_server/test_spawn_server.rb | 8 ++++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 993a2243f..82496c898 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -594,9 +594,7 @@ def log_upload_file(socket, parameters) def log_upload_sanitize_path(path) log_path = Pathname(app.log_dir) full_path = path.realpath(log_path) - if full_path.to_s.start_with?(log_path.to_s + "/") - return full_path - end + return full_path if full_path.to_s.start_with?(log_path.to_s + "/") raise ArgumentError, "cannot upload files not within the app's log directory" diff --git a/test/roby_app/spawn_server/test_spawn_server.rb b/test/roby_app/spawn_server/test_spawn_server.rb index 5e1eb8533..ab8f9bec1 100644 --- a/test/roby_app/spawn_server/test_spawn_server.rb +++ b/test/roby_app/spawn_server/test_spawn_server.rb @@ -75,13 +75,17 @@ def upload_testfile it "rejects an invalid user" do ftp_open do |ftp| - assert_raises(Net::FTPPermError) { ftp.login("user", @password) } + assert_raises(Net::FTPPermError) do + ftp.login("user", @password) + end end end it "rejects an invalid password" do ftp_open do |ftp| - assert_raises(Net::FTPPermError) { ftp.login(@user, "password") } + assert_raises(Net::FTPPermError) do + ftp.login(@user, "password") + end end end From 4ce4e888a57f5c54c66230191661fd411b561ab8 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 6 Dec 2022 22:11:22 -0300 Subject: [PATCH 034/260] fix: use log_dir as default log transfer target dir --- lib/syskit/roby_app/plugin.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index 3ceec93ff..3be51a6e2 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -172,8 +172,9 @@ def syskit_log_transfer_setup "cannot enable log transfer without log rotation" end - @syskit_log_transfer_manager = - LogTransferManager.new(Syskit.conf.log_transfer) + conf = Syskit.conf.log_transfer + conf.target_dir ||= log_dir + @syskit_log_transfer_manager = LogTransferManager.new(conf) end def syskit_log_transfer_cleanup From 54bffe265fd0170b6a44611cde5a4656f4df0c2d Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 7 Dec 2022 10:21:52 -0300 Subject: [PATCH 035/260] fix: workaround ftpd setting Thread.abort_on_exception globally --- lib/syskit/roby_app/log_transfer_server/spawn_server.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb index f9f81c542..b72046da5 100644 --- a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb +++ b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb @@ -47,7 +47,10 @@ def initialize( server.log = make_log server.nat_ip = nat_ip @server = server + Thread.abort_on_exception = false @server.start + sleep 0.1 until Thread.abort_on_exception + Thread.abort_on_exception = false @port = @server.bound_port display_connection_info if verbose end From 67e48eecf089b75dfe9b51ef72a482678853e761 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Thu, 15 Dec 2022 11:53:04 -0300 Subject: [PATCH 036/260] fix: remove unnecessary base_setup/base_cleanup Roby.app.setup already calls base_setup/base_cleanup, so this was doing base_setup all over again. --- lib/syskit/cli/gen_main.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/syskit/cli/gen_main.rb b/lib/syskit/cli/gen_main.rb index cda9bc006..ebc79c9db 100644 --- a/lib/syskit/cli/gen_main.rb +++ b/lib/syskit/cli/gen_main.rb @@ -188,7 +188,6 @@ def profile(name) def orogen(project_name) Roby.app.require_app_dir(needs_current: true) Roby.app.single = true - Roby.app.base_setup Syskit.conf.only_load_models = true Roby.app.setup @@ -210,7 +209,6 @@ def orogen(project_name) ) ensure Roby.app.cleanup - Roby.app.base_cleanup end no_commands do From bece6716fb544bae862b27259a64e3b349cc86a9 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 15 Dec 2022 16:10:45 -0300 Subject: [PATCH 037/260] fix: remove log files after a successful upload --- .../roby_app/remote_processes/server.rb | 1 + .../remote_processes/test_remote_processes.rb | 20 +++++++++++-------- test/roby_app/test_log_transfer_manager.rb | 3 ++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 82496c898..47ce07fc4 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -619,6 +619,7 @@ def log_upload_state log_dir = Pathname.new(app.log_dir) results.each do |r| if r.success? + r.file.unlink r.file = Pathname.new(r.file).relative_path_from(log_dir).to_s end end diff --git a/test/roby_app/remote_processes/test_remote_processes.rb b/test/roby_app/remote_processes/test_remote_processes.rb index 0b1f2cce1..10e8e6ec3 100644 --- a/test/roby_app/remote_processes/test_remote_processes.rb +++ b/test/roby_app/remote_processes/test_remote_processes.rb @@ -309,9 +309,7 @@ @port, @certificate = spawn_log_transfer_server remote_app_path = Pathname(@remote_log_dir).each_child.first @logfile = remote_app_path / "logfile.log" - File.open(@logfile, "wb") do |f| - f.write(SecureRandom.random_bytes(547)) # create random 547 byte - end + create_logfile(547) end after do @@ -327,13 +325,12 @@ @user, @password, @logfile ) assert_upload_succeeds - assert_equal File.read(path), File.read(@logfile) + assert_equal @logfile_contents, File.read(path) + refute File.exist?(@logfile) end it "rate-limits the file transfer" do - File.open(@logfile, "wb") do |f| - f.write(SecureRandom.random_bytes(1024 * 1024)) - end + create_logfile(1024 * 1024) tic = Time.now client.log_upload_file( "localhost", @port, @certificate, @@ -348,7 +345,7 @@ "transfer took #{toc - tic} instead of the expected 2s" ) path = File.join(@temp_serverdir, "logfile.log") - assert_equal File.read(path), File.read(@logfile) + assert_equal @logfile_contents, File.read(path) end it "rejects a wrong user" do @@ -389,6 +386,8 @@ assert_equal @logfile, result.file refute result.success? assert_match(/File already exists/, result.message) + # Does not delete the file + assert File.file?(@logfile) end it "fails on an invalid certificate" do @@ -405,6 +404,11 @@ assert_match(/certificate verify failed/, result.message) end + def create_logfile(size) + @logfile.write(SecureRandom.random_bytes(size)) + @logfile_contents = @logfile.read + end + def spawn_log_transfer_server @temp_serverdir = make_tmpdir @user = "test.user" diff --git a/test/roby_app/test_log_transfer_manager.rb b/test/roby_app/test_log_transfer_manager.rb index af126058f..a8ee18dc2 100644 --- a/test/roby_app/test_log_transfer_manager.rb +++ b/test/roby_app/test_log_transfer_manager.rb @@ -141,6 +141,7 @@ def assert_upload_fails(client, match) end def assert_upload_succeeds(test_file_path, client) + expected_content = test_file_path.read result = assert_upload_with_single_result(@manager, client) assert result.success?, "transfer failed, message: #{result.message}" @@ -148,7 +149,7 @@ def assert_upload_succeeds(test_file_path, client) actual_content = File.read( File.join(@conf.target_dir, result.file) ) - assert_equal test_file_path.read, actual_content + assert_equal expected_content, actual_content end def assert_upload_with_single_result(manager, client) From 4575e685f2b1e86297b589aefc8b42a0a8c1af58 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 3 Jan 2023 17:09:36 -0300 Subject: [PATCH 038/260] feat: implement the --set option for `syskit doc` --- lib/syskit/cli/doc_main.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/syskit/cli/doc_main.rb b/lib/syskit/cli/doc_main.rb index a0217e90d..da755f901 100644 --- a/lib/syskit/cli/doc_main.rb +++ b/lib/syskit/cli/doc_main.rb @@ -19,6 +19,9 @@ class DocMain < Thor option :exclude, type: :array, default: [], desc: "list of path patterns to exclude from documentation" + option :set, + type: :array, default: [], + desc: "set some configuration parameters" def gen(target_path) MetaRuby.keep_definition_location = true roby_app_configure @@ -40,6 +43,8 @@ def roby_app end def roby_app_configure + apply_set_options(roby_app) + roby_app.require_app_dir roby_app.using "syskit" roby_app.development_mode = false @@ -49,6 +54,13 @@ def roby_app_configure roby_app.setup_for_minimal_tooling end + def apply_set_options(app) + (options[:set] || []).each do |kv| + app.argv_set << kv + Roby::Application.apply_conf_from_argv(kv) + end + end + def roby_autoload_orogen_projects (models_path / "orogen").glob("*.rb").each do |extension_file| project_name = extension_file.sub_ext("").basename.to_s From f833217f764d5b41c7b8922ae03915c0c9e43b2a Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 5 Jan 2023 17:31:34 -0300 Subject: [PATCH 039/260] fix: avoid creating two connections to the local process server --- lib/syskit/roby_app/plugin.rb | 23 +++++++++++++------ .../roby_app/remote_processes/client.rb | 8 +++---- test/roby_app/test_plugin.rb | 1 + 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index 3be51a6e2..6603785b7 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -150,8 +150,9 @@ def self.setup(app) !(app.single? && app.simulation?) if start_local_process_server - start_local_process_server(redirect: Syskit.conf.redirect_local_process_server?) - @local_process_server_client = create_local_process_server_client(app) + start_local_process_server( + redirect: Syskit.conf.redirect_local_process_server? + ) connect_to_local_process_server(app) else fake_client = Configuration::ModelOnlyServer.new(app.default_loader) @@ -238,8 +239,8 @@ def self.cleanup(app) end app.syskit_log_transfer_cleanup - disconnect_all_process_servers stop_local_process_server(app) + disconnect_all_process_servers end # Hook called by the main application to prepare for execution @@ -624,10 +625,16 @@ def self.has_local_process_server? @server_pid end + def self.connect_to_local_process_server(app) + @local_process_server_client = create_local_process_server_client(app) + register_local_process_server_client(@local_process_server_client, app) + @local_process_server_client + end + def self.create_local_process_server_client(app) unless @server_pid raise Syskit::RobyApp::RemoteProcesses::Client::StartupFailed, - "#connect_to_local_process_server got called but "\ + "#create_local_process_server_client got called but "\ "no process server is being started" end @@ -666,9 +673,7 @@ def self.create_local_process_server_client(app) client end - def self.connect_to_local_process_server(app) - client = create_local_process_server_client(app) - + def self.register_local_process_server_client(client, app) # Do *not* manage the log directory for that one ... conf = Syskit.conf.register_process_server("localhost", client, app.log_dir) conf.supports_log_transfer = true @@ -680,6 +685,10 @@ def self.connect_to_local_process_server(app) def self.stop_local_process_server(app) return unless has_local_process_server? + if Syskit.conf.has_process_server?("localhost") + Syskit.conf.remove_process_server("localhost") + end + @local_process_server_client ||= create_local_process_server_client(app) @local_process_server_client.quit_server diff --git a/lib/syskit/roby_app/remote_processes/client.rb b/lib/syskit/roby_app/remote_processes/client.rb index dc7e7614b..8f88de5d0 100644 --- a/lib/syskit/roby_app/remote_processes/client.rb +++ b/lib/syskit/roby_app/remote_processes/client.rb @@ -121,10 +121,6 @@ def info(timeout: @response_timeout) Marshal.load(socket) end - def disconnect - socket.close - end - class TimeoutError < RuntimeError end @@ -341,6 +337,10 @@ def quit_server socket.write(COMMAND_QUIT) end + def disconnect + socket.close + end + def close socket.close end diff --git a/test/roby_app/test_plugin.rb b/test/roby_app/test_plugin.rb index f2a6de755..7565e0203 100644 --- a/test/roby_app/test_plugin.rb +++ b/test/roby_app/test_plugin.rb @@ -60,6 +60,7 @@ def create_process_server(name) m1.orogen_model.find_task_by_name("m1task").task_model end end + describe "local process server startup" do before do Syskit.conf.remove_process_server "localhost" From 23c89b26c2c57e57a5c6336b8cae7e060f3f98f6 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Fri, 6 Jan 2023 15:48:49 -0300 Subject: [PATCH 040/260] fix: remove the now-unused `enabled` flag in log transfer configuration Transfer is enabled by setting the IP --- lib/syskit/roby_app/configuration.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index 0d59793ea..7ffb04369 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -52,9 +52,9 @@ class Configuration # Configuration of Syskit's log transfer functionality # # Minimum configuration: set `ip` to an IP which the process servers - # can reach and set `enabled` to true. You must also configure log rotation - # ({#log_rotation_period}). Syskit will transfer the rotated logs to the - # main Syskit's instance log directory. + # can reach. You must also configure log rotation + # ({#log_rotation_period}). Syskit will transfer the rotated logs to + # the main Syskit's instance log directory. # # If you want to transfer to another dir, also set {#target_dir}. If you do # set {#target_dir}, local files will also be transferred. There is currently @@ -146,7 +146,6 @@ def initialize(app) @log_rotation_period = nil @log_transfer = LogTransferManager::Configuration.new( - enabled: false, user: "syskit", port: 22, password: SecureRandom.base64(32), From 6555bdd7e8b35c6334a08e1f02df7d8a6b4f648f Mon Sep 17 00:00:00 2001 From: Sylvain Date: Fri, 6 Jan 2023 15:49:41 -0300 Subject: [PATCH 041/260] fix: set the self-spawned FTP server interface to the configured IP `interface` is the IP the server is binding itself to, which should logically be the one the process servers will try to connect. It was currently always 127.0.0.1, which let the tests pass but fails in practice --- lib/syskit/roby_app/log_transfer_manager.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/syskit/roby_app/log_transfer_manager.rb b/lib/syskit/roby_app/log_transfer_manager.rb index 77f1a2ef7..e96aaa06e 100644 --- a/lib/syskit/roby_app/log_transfer_manager.rb +++ b/lib/syskit/roby_app/log_transfer_manager.rb @@ -35,18 +35,22 @@ def dispose(process_servers, flush: true) def server_start raise ArgumentError, "log transfer server already running" if @server - @self_signed_ca = TmpRootCA.new(@conf.ip) - @conf.user ||= "Syskit" - @conf.password ||= SecureRandom.base64(32) - @conf.certificate = @self_signed_ca.certificate - + server_update_self_spawned_conf @server = LogTransferServer::SpawnServer.new( @conf.target_dir, @conf.user, @conf.password, - @self_signed_ca.private_certificate_path + @self_signed_ca.private_certificate_path, + interface: @conf.ip ) @conf.port = @server.port end + def server_update_self_spawned_conf + @self_signed_ca = TmpRootCA.new(@conf.ip) + @conf.user ||= "Syskit" + @conf.password ||= SecureRandom.base64(32) + @conf.certificate = @self_signed_ca.certificate + end + # Whether files from the given directory should be transferred def transfer_local_files_from?(dir) @conf.target_dir != dir From 2a40e4ccb9ae86e8838b7f6a75c34cac3db95340 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Fri, 6 Jan 2023 15:50:45 -0300 Subject: [PATCH 042/260] fix: periodically poll the log transfer state It is actually expected by the server (this is where the transferred files are removed), but also will finally give some feedback about what was happening - debugging this has been a pain. --- lib/syskit/roby_app/plugin.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index 3be51a6e2..db2565340 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -195,6 +195,21 @@ def syskit_log_transfer_process_servers end end + def syskit_log_transfer_poll_state + syskit_log_transfer_process_servers.each do |process_server_config| + result = process_server_config.client.log_upload_state + ::Robot.info "#{result.pending_count} log transfers pending or in "\ + "progress from #{process_server_config.name}" + result.each_result do |r| + if r.success + ::Robot.info " transferred #{r.file}" + else + ::Robot.info " transfer of #{r.file} failed: #{r.message}" + end + end + end + end + # Rotate logs, and transfer the old logs if log transfer is configured def syskit_log_perform_rotation_and_transfer rotated_logs = syskit_rotate_logs @@ -249,6 +264,7 @@ def self.prepare(app) if Syskit.conf.log_rotation_period app.execution_engine.every(Syskit.conf.log_rotation_period) do app.syskit_log_perform_rotation_and_transfer + app.syskit_log_transfer_poll_state end end end From 0cc71910d5cf91b9c8e5b98f8502f8a7e2216010 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 12 Jan 2023 17:39:05 -0300 Subject: [PATCH 043/260] fix(cli): change `doc gen --set` to behave like in `run` Arrays in thor expected to be called like `--set=one two three`, while `run` expects to have multiple `--set` options. Luckily, thor handles the second behavior with `repeatable: true` --- lib/syskit/cli/doc_main.rb | 2 +- test/cli/test_doc_main.rb | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 test/cli/test_doc_main.rb diff --git a/lib/syskit/cli/doc_main.rb b/lib/syskit/cli/doc_main.rb index da755f901..7d75d2698 100644 --- a/lib/syskit/cli/doc_main.rb +++ b/lib/syskit/cli/doc_main.rb @@ -20,7 +20,7 @@ class DocMain < Thor type: :array, default: [], desc: "list of path patterns to exclude from documentation" option :set, - type: :array, default: [], + type: :string, repeatable: true, desc: "set some configuration parameters" def gen(target_path) MetaRuby.keep_definition_location = true diff --git a/test/cli/test_doc_main.rb b/test/cli/test_doc_main.rb new file mode 100644 index 000000000..5f9930208 --- /dev/null +++ b/test/cli/test_doc_main.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require "syskit/test/self" +require "roby/test/aruba_minitest" + +module Syskit + module CLI + describe "syskit gen" do + include Roby::Test::ArubaMinitest + + before do + run_command_and_stop "syskit gen app" + + @target_path = make_tmppath + end + + it "fails if the config does not load" do + robot_config requires: <<~RUBY + raise + RUBY + + cmd = doc_gen fail_on_error: false + refute_equal 0, cmd.exit_status + end + + it "allows multiple --set arguments" do + robot_config requires: <<~RUBY + raise if !Conf.set1? || !Conf.set2? + RUBY + + doc_gen "--set", "set1=true", "--set", "set2=true" + end + + def doc_gen(*args, fail_on_error: true) + run_command_and_stop "syskit doc gen #{@target_path} #{args.join(' ')}", + fail_on_error: fail_on_error + end + + def robot_config(requires: "") + contents = <<~RUBY + Robot.requires do + #{requires} + end + RUBY + write_file "config/robots/default.rb", contents + end + end + end +end From 53e7b72979b8f02aa2f530b3a4193b562df2708e Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sat, 4 Feb 2023 21:41:54 -0300 Subject: [PATCH 044/260] fix: default ftp server port 22 makes ... no sense obviously --- lib/syskit/roby_app/configuration.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index 7ffb04369..a6fcf895f 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -147,7 +147,7 @@ def initialize(app) @log_rotation_period = nil @log_transfer = LogTransferManager::Configuration.new( user: "syskit", - port: 22, + port: 20_301, password: SecureRandom.base64(32), self_spawned: true, certificate: nil, # Use random generated self-signed certificate From 9250ee598c564216b5e93393b83641e9ff8e2acb Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sat, 4 Feb 2023 21:49:22 -0300 Subject: [PATCH 045/260] fix: make implicit/explicit FTPs configurable Implicit FTPs support is broken on Ruby 2.5. The current implementation was switching based on whether the current ruby is 2.5 or 2.7, but that did not account for mixed systems, where the main syskit would run one version and process servers another. Essentially, one should use implicit ftps if all rubies are 2.7+, and not use it otherwise. --- lib/syskit/roby_app/configuration.rb | 3 ++- lib/syskit/roby_app/log_transfer_manager.rb | 12 +++++++++--- .../roby_app/log_transfer_server/spawn_server.rb | 4 ++-- lib/syskit/roby_app/remote_processes/client.rb | 5 +++-- lib/syskit/roby_app/remote_processes/ftp_upload.rb | 6 ++++-- lib/syskit/roby_app/remote_processes/server.rb | 7 ++++--- test/roby_app/spawn_server/test_spawn_server.rb | 7 +++++-- test/roby_app/test_log_transfer_manager.rb | 3 ++- test/roby_app/test_plugin.rb | 4 +++- 9 files changed, 34 insertions(+), 17 deletions(-) diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index a6fcf895f..567a64eb3 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -153,7 +153,8 @@ def initialize(app) certificate: nil, # Use random generated self-signed certificate target_dir: nil, # Use the app's log dir default_max_upload_rate: Float::INFINITY, - max_upload_rates: {} + max_upload_rates: {}, + implicit_ftps: LogTransferServer.use_implicit_ftps? ) clear diff --git a/lib/syskit/roby_app/log_transfer_manager.rb b/lib/syskit/roby_app/log_transfer_manager.rb index e96aaa06e..b4a5fdfb6 100644 --- a/lib/syskit/roby_app/log_transfer_manager.rb +++ b/lib/syskit/roby_app/log_transfer_manager.rb @@ -39,7 +39,8 @@ def server_start @server = LogTransferServer::SpawnServer.new( @conf.target_dir, @conf.user, @conf.password, @self_signed_ca.private_certificate_path, - interface: @conf.ip + interface: @conf.ip, + implicit_ftps: @conf.implicit_ftps? ) @conf.port = @server.port end @@ -79,7 +80,8 @@ def transfer_one_process_server_logs(process_server, paths) process_server.client.log_upload_file( @conf.ip, @conf.port, @conf.certificate, @conf.user, @conf.password, Pathname(path), - max_upload_rate: upload_rate + max_upload_rate: upload_rate, + implicit_ftps: @conf.implicit_ftps? ) end end @@ -141,7 +143,7 @@ def server_stop Configuration = Struct.new( :enabled, :ip, :port, :user, :password, :certificate, :self_spawned, :target_dir, :default_max_upload_rate, - :max_upload_rates, + :max_upload_rates, :implicit_ftps, keyword_init: true ) do def enabled? @@ -152,6 +154,10 @@ def self_spawned? self_spawned end + def implicit_ftps? + implicit_ftps + end + # Return the upload rate limit for a given process server # # If {#max_upload_rate} contains an entry for this process server diff --git a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb index b72046da5..1405d5c57 100644 --- a/lib/syskit/roby_app/log_transfer_server/spawn_server.rb +++ b/lib/syskit/roby_app/log_transfer_server/spawn_server.rb @@ -26,7 +26,7 @@ def initialize( password, certfile_path, interface: "127.0.0.1", - tls: LogTransferServer.use_implicit_ftps? ? :implicit : :explicit, + implicit_ftps: LogTransferServer.use_implicit_ftps?, port: 0, session_timeout: default_session_timeout, nat_ip: nil, @@ -39,7 +39,7 @@ def initialize( server = Ftpd::FtpServer.new(driver) server.interface = interface server.port = port - server.tls = tls + server.tls = implicit_ftps ? :implicit : :explicit server.passive_ports = passive_ports server.certfile_path = certfile_path server.auth_level = Ftpd.const_get("AUTH_PASSWORD") diff --git a/lib/syskit/roby_app/remote_processes/client.rb b/lib/syskit/roby_app/remote_processes/client.rb index 8f88de5d0..2ea47df52 100644 --- a/lib/syskit/roby_app/remote_processes/client.rb +++ b/lib/syskit/roby_app/remote_processes/client.rb @@ -229,12 +229,13 @@ def queue_death_announcement # upload progress def log_upload_file( host, port, certificate, user, password, localfile, - max_upload_rate: Float::INFINITY + max_upload_rate: Float::INFINITY, + implicit_ftps: LogTransferServer.use_implicit_ftps? ) socket.write(COMMAND_LOG_UPLOAD_FILE) Marshal.dump( [host, port, certificate, user, password, localfile, - max_upload_rate], socket + max_upload_rate, implicit_ftps], socket ) wait_for_ack diff --git a/lib/syskit/roby_app/remote_processes/ftp_upload.rb b/lib/syskit/roby_app/remote_processes/ftp_upload.rb index 94cf2afec..b145464c2 100644 --- a/lib/syskit/roby_app/remote_processes/ftp_upload.rb +++ b/lib/syskit/roby_app/remote_processes/ftp_upload.rb @@ -7,7 +7,8 @@ module RemoteProcesses class FTPUpload def initialize( # rubocop:disable Metrics/ParameterLists host, port, certificate, user, password, file, - max_upload_rate: Float::INFINITY + max_upload_rate: Float::INFINITY, + implicit_ftps: false ) @host = host @@ -18,6 +19,7 @@ def initialize( # rubocop:disable Metrics/ParameterLists @file = file @max_upload_rate = Float(max_upload_rate) + @implicit_ftps = implicit_ftps end # Create a temporary file with the FTP server's public key, to pass @@ -40,7 +42,7 @@ def open Net::FTP.open( @host, private_data_connection: false, port: @port, - implicit_ftps: LogTransferServer.use_implicit_ftps?, + implicit_ftps: @implicit_ftps, ssl: { verify_mode: OpenSSL::SSL::VERIFY_PEER, ca_file: cert_path } ) do |ftp| diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 47ce07fc4..60c38cf5f 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -569,8 +569,8 @@ def quit end def log_upload_file(socket, parameters) - host, port, certificate, user, password, localfile, max_upload_rate = - parameters + host, port, certificate, user, password, localfile, + max_upload_rate, implicit_ftps = parameters debug "#{socket} requested uploading of #{localfile}" @@ -587,7 +587,8 @@ def log_upload_file(socket, parameters) FTPUpload.new( host, port, certificate, user, password, localfile, - max_upload_rate: max_upload_rate || Float::INFINITY + max_upload_rate: max_upload_rate || Float::INFINITY, + implicit_ftps: implicit_ftps ) end diff --git a/test/roby_app/spawn_server/test_spawn_server.rb b/test/roby_app/spawn_server/test_spawn_server.rb index ab8f9bec1..fe24f8e15 100644 --- a/test/roby_app/spawn_server/test_spawn_server.rb +++ b/test/roby_app/spawn_server/test_spawn_server.rb @@ -18,9 +18,12 @@ def spawn_server private_key_path = File.join( __dir__, "..", "remote_processes", "cert-private.crt" ) + + @implicit_ftps = LogTransferServer.use_implicit_ftps? @server = SpawnServer.new( @temp_serverdir, @user, @password, - private_key_path + private_key_path, + implicit_ftps: @implicit_ftps ) end @@ -28,7 +31,7 @@ def ftp_open(certfile_path: @certfile_path, &block) Net::FTP.open( "localhost", port: @server.port, - implicit_ftps: LogTransferServer.use_implicit_ftps?, + implicit_ftps: @implicit_ftps, ssl: { verify_mode: OpenSSL::SSL::VERIFY_PEER, verify_hostname: false, ca_file: certfile_path }, diff --git a/test/roby_app/test_log_transfer_manager.rb b/test/roby_app/test_log_transfer_manager.rb index a8ee18dc2..862d1358d 100644 --- a/test/roby_app/test_log_transfer_manager.rb +++ b/test/roby_app/test_log_transfer_manager.rb @@ -15,7 +15,8 @@ module RobyApp @conf = LogTransferManager::Configuration.new( ip: "127.0.0.1", self_spawned: true, - max_upload_rates: {} + max_upload_rates: {}, + implicit_ftps: LogTransferServer.use_implicit_ftps? ) @conf.target_dir = make_tmpdir @manager = nil diff --git a/test/roby_app/test_plugin.rb b/test/roby_app/test_plugin.rb index 7565e0203..85b155727 100644 --- a/test/roby_app/test_plugin.rb +++ b/test/roby_app/test_plugin.rb @@ -278,6 +278,7 @@ def rotate_log Syskit.conf.log_transfer.password = "pass" Syskit.conf.log_transfer.certificate = "cert" Syskit.conf.log_transfer.port = 42 + Syskit.conf.log_transfer.implicit_ftps = false conf = Syskit.conf.process_server_config_for("localhost") flexmock(conf).should_receive(supports_log_transfer?: true) flexmock(app) @@ -292,7 +293,8 @@ def rotate_log .should_receive(:log_upload_file).explicitly .with("127.0.0.1", 42, "cert", "user", "pass", Pathname("old_log_file.log"), - max_upload_rate: Float::INFINITY) + max_upload_rate: Float::INFINITY, + implicit_ftps: false) .once app.syskit_log_perform_rotation_and_transfer end From 533fa854152b01c536fd1c1aa9775fb878a8449e Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 15 Feb 2023 10:05:20 -0300 Subject: [PATCH 046/260] fix: use the actual name instead of an undeclared variable This was probably missed during refactoring, `m` isnt declared anywhere. This would cause a stack too deep error. --- .../actions/interface_model_extension.rb | 4 ++-- .../actions/test_interface_model_extension.rb | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/syskit/actions/interface_model_extension.rb b/lib/syskit/actions/interface_model_extension.rb index d042c6f24..65b634f41 100644 --- a/lib/syskit/actions/interface_model_extension.rb +++ b/lib/syskit/actions/interface_model_extension.rb @@ -223,13 +223,13 @@ def profile def has_through_method_missing?(name) MetaRuby::DSLs.has_through_method_missing?( - profile, m, "_tag" => :has_tag? + profile, name, "_tag" => :has_tag? ) || super end def find_through_method_missing(name, args) MetaRuby::DSLs.find_through_method_missing( - profile, m, args, "_tag" => :find_tag + profile, name, args, "_tag" => :find_tag ) || super end diff --git a/test/actions/test_interface_model_extension.rb b/test/actions/test_interface_model_extension.rb index a38f5d1a3..34225bdb3 100644 --- a/test/actions/test_interface_model_extension.rb +++ b/test/actions/test_interface_model_extension.rb @@ -308,6 +308,27 @@ def call_action_method(**arguments, &block) end end + describe "method missing" do + before do + @action_m = Roby::Actions::Interface.new_submodel + @srv_m = Syskit::DataService.new_submodel + profile_m = Syskit::Actions::Profile.new + profile_m.tag "t", @srv_m + + @action_m.use_profile profile_m + end + + it "returns an used profile's tag model when accessed with _tag "\ + "at the model level" do + assert @action_m.t_tag.fullfills?(@srv_m) + end + + it "returns an used profile's tag model when accessed with _tag "\ + "at the instance level" do + assert @action_m.new(plan).t_tag.fullfills?(@srv_m) + end + end + describe "overloading of definitions by actions" do attr_reader :actions, :profile before do From a3b8eec3743d330fb24f092d994b9b62577fb275 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 16 Feb 2023 16:35:22 -0300 Subject: [PATCH 047/260] feat(test): allow passing read_only to stubbed deployments --- lib/syskit/test/network_manipulation.rb | 38 +++++++++++++++++++------ 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/lib/syskit/test/network_manipulation.rb b/lib/syskit/test/network_manipulation.rb index 3a5160d42..f4ad9329e 100644 --- a/lib/syskit/test/network_manipulation.rb +++ b/lib/syskit/test/network_manipulation.rb @@ -284,14 +284,26 @@ def syskit_stub_task_context_model(name, &block) model end - def syskit_stub_configured_deployment( - task_model = nil, - task_name = syskit_default_stub_name(task_model), + # Create a ready-to-deployed object for the given task model + # + # @param [Syskit::TaskContext] task_model + # @param [String] task_name + # @param [Boolean] remote_task stubbed tasks are always ruby tasks, i.e. + # tasks that run inside the Syskit process. This flag controls whether + # the task should be handled as if they were remote, or as ruby tasks. + # This is meant to be used during Syskit's own tests, to ensure we don't + # rely on ruby task behavior when testing remote task-handling code + # @param [Boolean] register the configured deployment in the test group + # This makes it available to further deployments + # @return [Syskit::Models::ConfiguredDeployment] + def syskit_stub_configured_deployment( # rubocop:disable Metrics/ParameterLists + task_model = nil, task_name = syskit_default_stub_name(task_model), remote_task: syskit_stub_resolves_remote_tasks?, - register: true, &block + register: true, read_only: [], &block ) configured_deployment = @__stubs.stub_configured_deployment( - task_model, task_name, remote_task: remote_task, &block + task_model, task_name, + read_only: read_only, remote_task: remote_task, &block ) if register @__test_deployment_group @@ -320,14 +332,22 @@ def syskit_stub_deployment_model( ) end - # Create a new stub deployment instance, optionally stubbing the - # model as well - def syskit_stub_deployment( + # Create a new stub deployment instance + # + # @param [Syskit::Models::Deployment] deployment_model the deployment model + # to use, or nil if the method should stub one + # @param [Syskit::Models::TaskContext] task_model set to a desired task + # context model to deploy when creating a new stub deployment model (i.e. + # when deployment_model is nil) + # @param [Boolean] read_only (see #syskit_stub_configured_deployment) + def syskit_stub_deployment( # rubocop:disable Metrics/ParameterLists name = "deployment", deployment_model = nil, + task_model: nil, read_only: [], remote_task: syskit_stub_resolves_remote_tasks?, &block ) deployment_model ||= syskit_stub_configured_deployment( - nil, name, remote_task: remote_task, &block + task_model, name, + read_only: read_only, remote_task: remote_task, &block ) task = deployment_model.new(process_name: name, on: "stubs") plan.add_permanent_task(task) From 00e7a8b6c9f87912a27792a1aa0819339c282f56 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Mon, 13 Feb 2023 17:12:08 -0300 Subject: [PATCH 048/260] fix(runtime): do not attempt to clean read_only tasks during deployment stop Deployments, when stopped cleanly, will also try to cleanup tasks that have been configured. This is obviously something we do not want for a read-only task. --- lib/syskit/deployment.rb | 14 ++++++++++---- test/test_deployment.rb | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/syskit/deployment.rb b/lib/syskit/deployment.rb index c3d2a2606..2da1648f9 100644 --- a/lib/syskit/deployment.rb +++ b/lib/syskit/deployment.rb @@ -213,9 +213,9 @@ def create_deployed_task( "expected #{base_syskit_task_model} or one of its subclasses" end - is_read_only_task = read_only.include?(mapped_name) - plan.add(task = syskit_task_model - .new(orocos_name: mapped_name, read_only: is_read_only_task)) + task = syskit_task_model + .new(orocos_name: mapped_name, read_only: read_only?(mapped_name)) + plan.add(task) task.executed_by self if scheduler_task task.depends_on scheduler_task, role: "scheduler" @@ -228,6 +228,10 @@ def create_deployed_task( task end + def read_only?(mapped_name) + read_only.include?(mapped_name) + end + # Returns an task instance that represents the given task in this # deployment. # @@ -818,7 +822,9 @@ def stop_prepare def stop_cleanly(promise, remote_task_handles) promise.then(description: "#{self}.stop_event - cleaning RTT tasks") do - remote_task_handles.each_value do |remote_task| + remote_task_handles.each do |mapped_name, remote_task| + next if read_only?(mapped_name) + begin if remote_task.handle.rtt_state == :STOPPED remote_task.handle.cleanup(false) diff --git a/test/test_deployment.rb b/test/test_deployment.rb index 90acb7f78..ceda7712f 100644 --- a/test/test_deployment.rb +++ b/test/test_deployment.rb @@ -236,6 +236,21 @@ def mock_raw_port(task, port_name) assert_equal true, task.read_only end + it "does not cleanup read-only tasks on shutdown" do + task_m = Syskit::TaskContext.new_submodel + deployment = syskit_stub_deployment( + "test", task_model: task_m, read_only: %w[test] + ) + expect_execution { deployment.start! }.to { emit deployment.ready_event } + + task = deployment.task("test") + Orocos.allow_blocking_calls do + task.orocos_task.configure + end + flexmock(task.orocos_task).should_receive(:cleanup).never + expect_execution { deployment.stop! }.to { emit deployment.stop_event } + end + describe "slave tasks" do before do @task_m = TaskContext.new_submodel do From 98ecfff06768355f53a1052d1d558db2849deb3a Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 16 Feb 2023 16:38:43 -0300 Subject: [PATCH 049/260] fix(runtime): handle reconfiguration of a readonly task by a remote syskit --- lib/syskit/runtime/update_task_states.rb | 2 ++ test/test_deployment.rb | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/lib/syskit/runtime/update_task_states.rb b/lib/syskit/runtime/update_task_states.rb index 21001ec94..efc0512fc 100644 --- a/lib/syskit/runtime/update_task_states.rb +++ b/lib/syskit/runtime/update_task_states.rb @@ -59,6 +59,8 @@ def self.handle_task_runtime_states(task) received_states = [] while (!state || task.orocos_task.runtime_state?(state)) && (new_state = task.update_orogen_state) + next if new_state == :PRE_OPERATIONAL && task.read_only? + received_states << new_state # Returns nil if we have a communication problem. In this diff --git a/test/test_deployment.rb b/test/test_deployment.rb index ceda7712f..8b57f4592 100644 --- a/test/test_deployment.rb +++ b/test/test_deployment.rb @@ -251,6 +251,30 @@ def mock_raw_port(task, port_name) expect_execution { deployment.stop! }.to { emit deployment.stop_event } end + it "handles a readonly task being stopped and cleaned up in a single cycle" do + # The bug this tests against was Syskit receiving stopped and + # pre operational in a single cycle. TaskContext#handle_state_changes + # would try to find an event for that transition and boom + task_m = Syskit::TaskContext.new_submodel + deployment = syskit_stub_deployment( + "test", task_model: task_m, read_only: %w[test] + ) + expect_execution { deployment.start! }.to { emit deployment.ready_event } + + task = deployment.task("test") + Orocos.allow_blocking_calls do + task.orocos_task.configure + task.orocos_task.start + end + expect_execution { task.start! }.to { emit task.start_event } + + Orocos.allow_blocking_calls do + task.orocos_task.stop + task.orocos_task.cleanup + end + execute_one_cycle + end + describe "slave tasks" do before do @task_m = TaskContext.new_submodel do From 1b60de33c3d95b05011ac7c4ba0eab27cd3e0cbf Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 15 Feb 2023 10:01:56 -0300 Subject: [PATCH 050/260] fix: properly handle action methods with required arguments in profile assertions The profile assertion action resolving methods try to instanciate method tasks, but was either (1) generating an error if the action method has missing required arguments *and* the action was toplevel or (2) generating an error claiming we could pass it to `exclude` - but we actually could not because of a bug *and* it was not the right suggestion. Fix both cases, add documentation and tests. On the exclude bug: Syskit tests that are given actions are instanciating the actions to figure out all the syskit definitions that are involved. But the same code path was used for the `exclude` argument (instanciate and then convert back to an action model), even though the exclusion is not recursive (i.e. we only want the model associated with the excluded object, not whatever action is derived from it). --- lib/syskit/test/profile_assertions.rb | 180 ++++++++++++++---- test/test/test_profile_assertions.rb | 258 ++++++++++++++++++++++++++ 2 files changed, 398 insertions(+), 40 deletions(-) diff --git a/lib/syskit/test/profile_assertions.rb b/lib/syskit/test/profile_assertions.rb index 6aa00259a..ae98d2d06 100644 --- a/lib/syskit/test/profile_assertions.rb +++ b/lib/syskit/test/profile_assertions.rb @@ -31,6 +31,14 @@ def pretty_print(pp) end end + # @api private + # + # Find all planning tasks in a task hierarchy and return the resolved actions + # + # @param [Roby::Task] task the hierarchy's root task + # @return [Array] the resolved actions. The method + # finds tasks with an action planner, syskit requirement tasks as well + # as expands state machines to find actions related to states. def resolve_actions_from_plan(task) actions = [] queue = [task] @@ -51,11 +59,59 @@ def resolve_actions_from_plan(task) actions end + # @api private + # # Validates an argument that can be an action, an action collection # (e.g. a profile) or an array of action, and normalizes it into an # array of actions # # @raise [ArgumentError] if the argument is invalid + def ActionModels(arg) + if arg.respond_to?(:each_action) + ActionModels(arg.each_action) + elsif arg.respond_to?(:to_action) + [arg.to_action.model] + elsif arg.respond_to?(:flat_map) + arg.flat_map { |a| ActionModels(a) } + elsif arg.respond_to?(:to_instance_requirements) + [Actions::Model::Action.new(arg)] + else + raise ArgumentError, + "expected an action or a collection of actions, but got "\ + "#{arg} of class #{arg.class}" + end + end + + # @api private + # + # Helper to {#Actions} to resolve a list of actions from a method action + # + # It returns either the actions that can be found from the instanciated + # method action, or itself if it has missing required arguments + # + # @param [Roby::Actions::Action] arg the method action + # @return [Array<#to_action>] the resolved actions + def actions_from_method_action(arg) + return [arg] if arg.has_missing_required_arg? + + plan = Roby::Plan.new + task = arg.instanciate(plan) + + # Now find the new actions + resolve_actions_from_plan(task).flat_map { |a| Actions(a) } + end + + # @api private + # + # Resolves all reachable syskit actions from an argument that can be + # an action, an action collection (e.g. a profile or action interface), + # or an array of actions + # + # The method returns as-is method actions that cannot be instanciated + # because of missing required arguments. Use {#BulkAssertAtomicActions} to + # filter these out automatically + # + # @raise [ArgumentError] if the argument is invalid def Actions(arg) if arg.respond_to?(:each_action) arg.each_action.flat_map do |a| @@ -67,12 +123,7 @@ def Actions(arg) return [arg] end - plan = Roby::Plan.new - task = arg.instanciate(plan) - - # Now find the new actions - resolve_actions_from_plan(task) - .flat_map { |a| Actions(a) } + actions_from_method_action(arg) elsif arg.respond_to?(:flat_map) arg.flat_map { |a| Actions(a) } elsif arg.respond_to?(:to_instance_requirements) @@ -100,7 +151,7 @@ def AtomicActions(arg) # should be ignored. Actions are compared on the basis of their # model (arguments do not count) def BulkAssertAtomicActions(arg, exclude: []) - exclude = Actions(exclude).map(&:model) + exclude = ActionModels(exclude) skipped_actions = [] actions = AtomicActions(arg).find_all do |action| if exclude.include?(action.model) @@ -119,22 +170,35 @@ def BulkAssertAtomicActions(arg, exclude: []) [actions, skipped_actions] end - # Tests that a definition or all definitions of a profile are - # self-contained, that is that the only variation points in the - # profile are profile tags. + # Tests that one or many syskit definitions are self-contained # - # If given a profile as argument, or no profile at all, will test on - # all definitions of resp. the given profile or the test's subject + # When it is part of a profile, a definition is self-contained if it only + # contains concrete component models or tags of said profile # # Note that it is a really good idea to maintain this property. No. - # Seriously. Keep it in your tests. + # Seriously. Keep it in your profile tests. + # + # When resolving actions that are not directly defined from profile + # definitions, the method will attempte to resolve method action by + # calling them. If there is a problem, pass the action model to the + # `exclude` argument. + # + # In particular, in the presence of action methods with required + # arguments, run one assert first with the action method excluded and + # another with that action and sample arguments. + # + # @param action_or_profile if an action interface or profile, test all + # definitions that are reachable from it. In the case of action interfaces, + # this means looking into method actions and action state machines. def assert_is_self_contained( action_or_profile = subject_syskit_model, message: "%s is not self contained", exclude: [], **instanciate_options ) actions = validate_actions(action_or_profile, exclude: exclude) do |skip| - flunk "could not validate some non-Syskit actions: "\ - "#{skip}, pass them to the 'exclude' argument to #{__method__}" + flunk "could not validate some non-Syskit actions: #{skip}, "\ + "probably because of required arguments. Pass the action to "\ + "the 'exclude' option of #{__method__}, and add a separate "\ + "assertion test with the arguments added explicitly" end actions.each do |act| @@ -211,26 +275,42 @@ def is_self_contained(action_or_profile = subject_syskit_model, **options) # Tests that the following definition can be successfully # instanciated in a valid, non-abstract network. # - # If given a profile, it will perform the test on each action of the - # profile taken in isolation. If you want to test whether actions - # can be instanciated at the same time, use - # {#assert_can_instanciate_together} + # When resolving actions that are not directly defined from profile + # definitions, the method will attempte to resolve method action by + # calling them. If there is a problem, pass the action model to the + # `exclude` argument. + # + # In particular, in the presence of action methods with required + # arguments, run one assert first with the action method excluded and + # another with that action and sample arguments. # - # If called without argument, it tests the spec's context profile + # @param action_or_profile if an action interface or profile, test all + # definitions that are reachable from it. In the case of action interfaces, + # this means looking into method actions and action state machines. + # @param together_with test that each single action in `action_or_profile` + # can be instanciated when all actions in `together_with` are instanciated + # at the same time. This can be used if the former depend on the presence + # of the latter, or if you want to test against conflicts. def assert_can_instanciate( action_or_profile = subject_syskit_model, exclude: [], together_with: [] ) actions = validate_actions(action_or_profile, exclude: exclude) do |skip| - flunk "could not validate some non-Syskit actions: "\ - "#{skip}, pass them to the 'exclude' argument to #{__method__}" + flunk "could not validate some non-Syskit actions: #{skip}, "\ + "probably because of required arguments. Pass the action to "\ + "the 'exclude' option of #{__method__}, and add a separate "\ + "assertion test with the arguments added explicitly" end - together_with = validate_actions(together_with) do |skip| - flunk "could not validate some non-Syskit actions given to "\ - "`together_with`: #{skip}, pass them to the 'exclude' "\ - "argument to #{__method__}" - end + together_with = + validate_actions(together_with, exclude: exclude) do |skip| + flunk "could not validate some non-Syskit actions given to "\ + "`together_with` in #{__method__}: #{skip}, "\ + "probably because of "\ + "missing arguments. If you are passing a profile or "\ + "action interface and do not require to test against "\ + "that particular action, pass it to the 'exclude' argument" + end actions.each do |action| tasks = assert_can_instanciate_together(action, *together_with) @@ -311,26 +391,42 @@ def expand_task_coordination_models(task) # that is they result in a valid, non-abstract network whose all # components have a deployment # - # If given a profile, it will perform the test on each action of the - # profile taken in isolation. If you want to test whether actions - # can be deployed at the same time, use - # {#assert_can_deploy_together} + # When resolving actions that are not directly defined from profile + # definitions, the method will attempte to resolve method action by + # calling them. If there is a problem, pass the action model to the + # `exclude` argument. # - # If called without argument, it tests the spec's context profile + # In particular, in the presence of action methods with required + # arguments, run one assert first with the action method excluded and + # another with that action and sample arguments. + # + # @param action_or_profile if an action interface or profile, test all + # definitions that are reachable from it. In the case of action interfaces, + # this means looking into method actions and action state machines. + # @param together_with test that each single action in `action_or_profile` + # can be instanciated when all actions in `together_with` are instanciated + # at the same time. This can be used if the former depend on the presence + # of the latter, or if you want to test against conflicts. def assert_can_deploy( action_or_profile = subject_syskit_model, exclude: [], together_with: [] ) actions = validate_actions(action_or_profile, exclude: exclude) do |skip| - flunk "could not validate some non-Syskit actions: "\ - "#{skip}, pass them to the 'exclude' argument to #{__method__}" + flunk "could not validate some non-Syskit actions: #{skip}, "\ + "probably because of required arguments. Pass the action to "\ + "the 'exclude' option of #{__method__}, and add a separate "\ + "assertion test with the arguments added explicitly" end - together_with = validate_actions(together_with) do |skip| - flunk "could not validate some non-Syskit actions given to "\ - "`together_with` #{skip}, pass them to the 'exclude' "\ - "argument to #{__method__}" - end + together_with = + validate_actions(together_with, exclude: exclude) do |skip| + flunk "could not validate some non-Syskit actions given to "\ + "`together_with` in #{__method__}: #{skip}, "\ + "probably because of "\ + "missing arguments. If you are passing a profile or action "\ + "interface and do not require to test against that "\ + "particular action, pass it to the 'exclude' argument" + end actions.each do |action| task = assert_can_deploy_together(action, *together_with) @@ -418,7 +514,11 @@ def can_configure_together(*actions) def validate_actions(action_or_profile, exclude: []) actions, skipped = BulkAssertAtomicActions(action_or_profile, exclude: exclude) - yield skipped.map(&:name).sort.join(", ") unless skipped.empty? + + unless skipped.empty? + action_names = "'#{skipped.map(&:name).sort.join("', '")}'" + yield(action_names) + end actions end diff --git a/test/test/test_profile_assertions.rb b/test/test/test_profile_assertions.rb index ebd19980c..bac57468a 100644 --- a/test/test/test_profile_assertions.rb +++ b/test/test/test_profile_assertions.rb @@ -15,12 +15,62 @@ module Test @cmp_m.add @srv_m, as: "test" end + describe "ActionModels" do + include ProfileAssertions + + before do + @profile_m = Syskit::Actions::Profile.new + @profile_m.define "test", @cmp_m + @interface_m = Roby::Actions::Interface.new_submodel + @interface_m.use_profile @profile_m + end + + it "resolves an instance requirements action" do + assert_equal [@interface_m.test_def.model], + ActionModels(@interface_m.test_def) + end + + it "resolves a method action" do + task_m = Roby::Task.new_submodel + @interface_m.class_eval do + describe("act").returns(task_m) + define_method :act do + end + end + + assert_equal [@interface_m.act.model], ActionModels(@interface_m.act) + end + + it "handles array of actions as argument" do + @interface_m.class_eval do + describe("act") + define_method(:act) {} + end + + assert_equal [@interface_m.act.model, @interface_m.test_def.model], + ActionModels([@interface_m.act, @interface_m.test_def]) + end + + it "resolves an action state machine" do + task_m = Roby::Task.new_submodel + @interface_m.class_eval do + describe("act").returns(task_m) + action_state_machine :act do + start state(task_m) + end + end + + assert_equal [@interface_m.act.model], ActionModels(@interface_m.act) + end + end + describe "Actions" do include ProfileAssertions before do @profile_m = Syskit::Actions::Profile.new @profile_m.define "test", @cmp_m + @profile_m.define "test2", @cmp_m @interface_m = Roby::Actions::Interface.new_submodel @interface_m.use_profile @profile_m end @@ -29,6 +79,21 @@ module Test assert_equal [@interface_m.test_def], Actions(@interface_m.test_def) end + it "accepts an array as argument" do + task_m = Roby::Task.new_submodel + @interface_m.class_eval do + describe("act").returns(task_m) + define_method(:act) do + root = task_m.new + root.depends_on(test2_def) + root + end + end + + assert_equal [@interface_m.test2_def, @interface_m.test_def], + Actions([@interface_m.act, @interface_m.test_def]) + end + it "resolves a method action that returns a task with "\ "a coordination model" do task_m = Roby::Task.new_submodel @@ -84,6 +149,91 @@ module Test end end + describe "BulkAssertAtomicActions" do + include ProfileAssertions + + before do + @profile_m = Syskit::Actions::Profile.new + @profile_m.define "test", @cmp_m + @interface_m = Roby::Actions::Interface.new_submodel + @interface_m.use_profile @profile_m + end + + it "lists all actions that can be resolved from its argument" do + @interface_m.describe(:sm) + @interface_m.action_state_machine :sm do + start state(test_def) + end + + assert_equal [[@interface_m.test_def], []], + BulkAssertAtomicActions(@interface_m.sm) + end + + it "excludes actions from the models listed in 'exclude'" do + @interface_m.describe(:sm) + @interface_m.action_state_machine :sm do + start state(test_def) + end + + found, skipped = BulkAssertAtomicActions( + @interface_m.sm, exclude: [@interface_m.test_def] + ) + assert_equal [], found + assert_equal [], skipped + end + + it "reports actions that cannot be instanciated "\ + "because of missing arguments" do + @interface_m.describe(:m_action).required_arg(:test, "some docs") + @interface_m.define_method(:m_action) do |test:| + end + + found, skipped = BulkAssertAtomicActions(@interface_m.m_action) + assert_equal [], found + assert_equal [@interface_m.m_action], skipped + end + + it "reports actions that cannot be instanciated "\ + "because of missing arguments within a state machine" do + @interface_m.describe(:m_action).required_arg(:test, "some docs") + @interface_m.define_method(:m_action) do |test:| + end + @interface_m.describe(:sm) + @interface_m.action_state_machine :sm do + start state(m_action) + end + + found, skipped = BulkAssertAtomicActions(@interface_m.sm) + assert_equal [], found + assert_equal [@interface_m.m_action], skipped + end + + it "lets the caller exclude atomic actions" do + @interface_m.describe(:sm) + @interface_m.action_state_machine :sm do + start state(test_def) + end + + found, skipped = BulkAssertAtomicActions( + @interface_m.sm, exclude: [@interface_m.sm] + ) + assert_equal [@interface_m.test_def], found + assert_equal [], skipped + end + + it "lets the caller exclude method actions with missing arguments" do + @interface_m.describe(:m_action).required_arg(:test, "") + @interface_m.define_method :m_action do |test:| + end + + found, skipped = BulkAssertAtomicActions( + @interface_m.m_action, exclude: [@interface_m.m_action] + ) + assert_equal [], found + assert_equal [], skipped + end + end + describe "assert_is_self_contained" do include ProfileAssertions @@ -136,6 +286,24 @@ module Test @test_profile.tag "test", @srv_m assert_is_self_contained(@cmp_m.use(@srv_m => @test_profile.test_tag)) end + + it "fails if some actions are not resolvable" do + flexmock(self) + .should_receive(:BulkAssertAtomicActions) + .with(action = flexmock, exclude: (excluded = flexmock)) + .and_return([[], + [flexmock(name: "some"), flexmock(name: "action")]]) + + e = assert_raises(Minitest::Assertion) do + assert_is_self_contained(action, exclude: excluded) + end + message = "could not validate some non-Syskit actions: 'action', "\ + "'some', probably because of required arguments. Pass "\ + "the action to the 'exclude' option of "\ + "assert_is_self_contained, and add a separate assertion "\ + "test with the arguments added explicitly" + assert_equal message, e.message + end end describe "assert_can_instanciate" do @@ -209,6 +377,51 @@ module Test assert_equal 9, t.bla end end + + it "fails if some actions are not resolvable" do + flexmock(self) + .should_receive(:BulkAssertAtomicActions) + .with(action = flexmock, exclude: (excluded = flexmock)) + .and_return([[], + [flexmock(name: "some"), flexmock(name: "action")]]) + + e = assert_raises(Minitest::Assertion) do + assert_can_instanciate(action, exclude: excluded) + end + message = "could not validate some non-Syskit actions: 'action', "\ + "'some', probably because of required arguments. Pass "\ + "the action to the 'exclude' option of "\ + "assert_can_instanciate, and add a separate assertion "\ + "test with the arguments added explicitly" + assert_equal message, e.message + end + + it "fails if some actions in together_with are not resolvable" do + action, together_with, exclude = 3.times.map { flexmock } + flexmock(self) + .should_receive(:BulkAssertAtomicActions) + .with(action, exclude: exclude) + .and_return([[], []]) + flexmock(self) + .should_receive(:BulkAssertAtomicActions) + .with(together_with, exclude: exclude) + .and_return([[], + [flexmock(name: "some"), flexmock(name: "action")]]) + + e = assert_raises(Minitest::Assertion) do + assert_can_instanciate( + action, exclude: exclude, together_with: together_with + ) + end + message = + "could not validate some non-Syskit actions given "\ + "to `together_with` in assert_can_instanciate: 'action', "\ + "'some', probably because of "\ + "missing arguments. If you are passing a profile or action "\ + "interface and do not require to test against that particular "\ + "action, pass it to the 'exclude' argument" + assert_equal message, e.message + end end describe "assert_can_deploy" do @@ -301,6 +514,51 @@ module Test .use_deployment(@deployment_m) ) end + + it "fails if some actions are not resolvable" do + flexmock(self) + .should_receive(:BulkAssertAtomicActions) + .with(action = flexmock, exclude: (excluded = flexmock)) + .and_return([[], + [flexmock(name: "some"), flexmock(name: "action")]]) + + e = assert_raises(Minitest::Assertion) do + assert_can_deploy(action, exclude: excluded) + end + message = "could not validate some non-Syskit actions: 'action', "\ + "'some', probably because of required arguments. Pass "\ + "the action to the 'exclude' option of "\ + "assert_can_deploy, and add a separate assertion "\ + "test with the arguments added explicitly" + assert_equal message, e.message + end + + it "fails if some actions in together_with are not resolvable" do + action, together_with, exclude = 3.times.map { flexmock } + flexmock(self) + .should_receive(:BulkAssertAtomicActions) + .with(action, exclude: exclude) + .and_return([[], []]) + flexmock(self) + .should_receive(:BulkAssertAtomicActions) + .with(together_with, exclude: exclude) + .and_return([[], + [flexmock(name: "some"), flexmock(name: "action")]]) + + e = assert_raises(Minitest::Assertion) do + assert_can_deploy( + action, exclude: exclude, together_with: together_with + ) + end + message = + "could not validate some non-Syskit actions given "\ + "to `together_with` in assert_can_deploy: 'action', "\ + "'some', probably because of "\ + "missing arguments. If you are passing a profile or action "\ + "interface and do not require to test against that particular "\ + "action, pass it to the 'exclude' argument" + assert_equal message, e.message + end end end end From 92d74cd2fa97acba0fa9c1b28a6e69094cae96d6 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 11 Apr 2023 09:23:01 -0300 Subject: [PATCH 051/260] fix: handle task.depends_on(composition) when scheduling reconfigurations While the normal data structure does not generate this type of patterns, they must be supported - it is generally speaking useful, and is already used by e.g. the transformer to represent dynamic transformation producers. The pattern obviously already triggered a bug since there was a specific bug fix for a task context->task context dependency. The fix was however too limited as it would not handle task context->composition. This change implements a pass after deployment where we would find the root components from the "old plan" that are not used in the "new plan", and cut the dependency relations between these two. This should really handle all cases. --- lib/syskit/network_generation/engine.rb | 50 ++++-- lib/syskit/network_generation/merge_solver.rb | 8 + test/network_generation/test_engine.rb | 148 +++++++++++------- 3 files changed, 140 insertions(+), 66 deletions(-) diff --git a/lib/syskit/network_generation/engine.rb b/lib/syskit/network_generation/engine.rb index d969caa7d..200999d9f 100644 --- a/lib/syskit/network_generation/engine.rb +++ b/lib/syskit/network_generation/engine.rb @@ -137,6 +137,8 @@ def apply_deployed_network_to_plan finalize_deployed_tasks end + sever_old_plan_from_new_plan + if @dataflow_dynamics @dataflow_dynamics.apply_merges(merge_solver) log_timepoint "apply_merged_to_dataflow_dynamics" @@ -147,6 +149,42 @@ def apply_deployed_network_to_plan end end + # "Cut" relations between the "old" plan and the new one + # + # At this stage, old components (task contexts and compositions) + # that are not part of the new plan may still be child of bits of + # the new plan. This happens if they are added as children of other + # task contexts. The transformer does this to register dynamic + # transformation producers + # + # This pass looks for all proxies of compositions and task contexts + # that are not the target of a merge operation. When this happens, + # we know that the component is not being reused, and we remove all + # dependency relations where it is child and where the parent is + # "useful" + # + # Note that we do this only for relations between Syskit + # components. Relations with "plan" Roby tasks are updated because + # we replace toplevel tasks. + def sever_old_plan_from_new_plan + old_tasks = + work_plan + .find_local_tasks(Syskit::Component) + .find_all(&:transaction_proxy?) + + merge_leaves = merge_solver.each_merge_leaf.to_set + old_tasks.each do |old_task| + next if merge_leaves.include?(old_task) + + parents = + old_task + .each_parent_task + .find_all { |t| merge_leaves.include?(t) } + + parents.each { |t| t.remove_child(old_task) } + end + end + class << self # Set of blocks registered with # register_instanciation_postprocessing @@ -602,9 +640,7 @@ def adapt_existing_deployment(deployment_task, existing_deployment_task) deployed_tasks.each do |task| existing_tasks = orocos_name_to_existing[task.orocos_name] || [] - unless existing_tasks.empty? - existing_task = find_current_deployed_task(existing_tasks) - end + existing_task = find_current_deployed_task(existing_tasks) if !existing_task || !task.can_be_deployed_by?(existing_task) debug do @@ -629,14 +665,6 @@ def adapt_existing_deployment(deployment_task, existing_deployment_task) "to finish before reconfiguring" end - parent_task_contexts = - previous_task - .each_parent_task - .find_all { |t| t.kind_of?(Syskit::TaskContext) } - - parent_task_contexts.each do |t| - t.remove_child(previous_task) - end new_task.should_configure_after(previous_task.stop_event) end existing_task = new_task diff --git a/lib/syskit/network_generation/merge_solver.rb b/lib/syskit/network_generation/merge_solver.rb index e59b750e4..ee2d57527 100644 --- a/lib/syskit/network_generation/merge_solver.rb +++ b/lib/syskit/network_generation/merge_solver.rb @@ -572,6 +572,14 @@ def display_merge_graph(title, merge_graph) break end end + + def each_merge_leaf + return enum_for(__method__) unless block_given? + + task_replacement_graph.each_vertex do |v| + yield(v) if task_replacement_graph.leaf?(v) + end + end end end end diff --git a/test/network_generation/test_engine.rb b/test/network_generation/test_engine.rb index a53c61644..98e9fa679 100644 --- a/test/network_generation/test_engine.rb +++ b/test/network_generation/test_engine.rb @@ -195,61 +195,6 @@ def work_plan .with_arguments(orocos_name: task.orocos_name).to_a assert_equal work_plan.wrap([task]), tasks end - - describe "when child of a composition" do - it "ensures that the existing deployment will be garbage collected" do - task_m = Syskit::TaskContext.new_submodel - cmp_m = Syskit::Composition.new_submodel - cmp_m.add task_m, as: "test" - - syskit_stub_configured_deployment(task_m) - cmp = syskit_deploy(cmp_m) - original_task = cmp.test_child - flexmock(task_m).new_instances.should_receive(:can_be_deployed_by?) - .with(->(proxy) { proxy.__getobj__ == cmp.test_child }).and_return(false) - new_cmp = syskit_deploy(cmp_m) - - # Should have instanciated a new composition since the children - # differ - refute_equal new_cmp, cmp - # Should have of course created a new task - refute_equal new_cmp.test_child, cmp.test_child - # And the old tasks should be ready to garbage-collect - assert_equal [cmp, original_task].to_set, - execute { plan.static_garbage_collect.to_set } - end - end - - describe "when child of a task" do - it "ensures that the existing deployment will be garbage collected" do - child_m = Syskit::TaskContext.new_submodel - parent_m = Syskit::TaskContext.new_submodel - parent_m.singleton_class.class_eval do - define_method(:instanciate) do |*args, **kw| - task = super(*args, **kw) - task.depends_on(child_m.instanciate(*args, **kw), - role: "test") - task - end - end - - syskit_stub_configured_deployment(child_m) - parent_m = syskit_stub_requirements(parent_m) - parent = syskit_deploy(parent_m) - child = parent.test_child - - flexmock(child_m).new_instances.should_receive(:can_be_deployed_by?) - .with(->(proxy) { proxy.__getobj__ == child }).and_return(false) - new_parent = syskit_deploy(parent_m) - new_child = new_parent.test_child - - assert_equal new_parent, parent - refute_equal new_child, child - # And the old tasks should be ready to garbage-collect - assert_equal [child].to_set, - execute { plan.static_garbage_collect.to_set } - end - end end describe "#adapt_existing_deployment" do @@ -343,6 +288,99 @@ def work_plan end end + describe "when scheduling tasks for reconfiguration" do + it "ensures that the old task is gargabe collected "\ + "when child of a composition" do + task_m = Syskit::TaskContext.new_submodel + cmp_m = Syskit::Composition.new_submodel + cmp_m.add task_m, as: "test" + + syskit_stub_configured_deployment(task_m) + cmp = syskit_deploy(cmp_m) + original_task = cmp.test_child + flexmock(task_m).new_instances.should_receive(:can_be_deployed_by?) + .with(->(proxy) { proxy.__getobj__ == cmp.test_child }).and_return(false) + new_cmp = syskit_deploy(cmp_m) + + # Should have instanciated a new composition since the children + # differ + refute_equal new_cmp, cmp + # Should have of course created a new task + refute_equal new_cmp.test_child, cmp.test_child + # And the old tasks should be ready to garbage-collect + assert_equal [cmp, original_task].to_set, + execute { plan.static_garbage_collect.to_set } + end + + it "ensures that the old task gets garbage collected when child "\ + "of another still useful task" do + child_m = Syskit::TaskContext.new_submodel + parent_m = Syskit::TaskContext.new_submodel + parent_m.singleton_class.class_eval do + define_method(:instanciate) do |*args, **kw| + task = super(*args, **kw) + task.depends_on(child_m.instanciate(*args, **kw), + role: "test") + task + end + end + + syskit_stub_configured_deployment(child_m) + parent_m = syskit_stub_requirements(parent_m) + parent = syskit_deploy(parent_m) + child = parent.test_child + + flexmock(child_m) + .new_instances.should_receive(:can_be_deployed_by?) + .with(->(proxy) { proxy.__getobj__ == child }).and_return(false) + new_parent = syskit_deploy(parent_m) + new_child = new_parent.test_child + + assert_equal new_parent, parent + refute_equal new_child, child + # And the old tasks should be ready to garbage-collect + assert_equal [child].to_set, + execute { plan.static_garbage_collect.to_set } + end + + it "ensures that the old task gets garbage collected when child "\ + "of a composition, itself child of a useful task" do + child_m = Syskit::TaskContext.new_submodel + cmp_m = Syskit::Composition.new_submodel + cmp_m.add child_m, as: "task" + parent_m = Syskit::TaskContext.new_submodel + parent_m.singleton_class.class_eval do + define_method(:instanciate) do |*args, **kw| + task = super(*args, **kw) + task.depends_on(cmp_m.instanciate(*args, **kw), + role: "test") + task + end + end + + syskit_stub_configured_deployment(child_m) + parent_m = syskit_stub_requirements(parent_m) + parent = syskit_deploy(parent_m) + child = parent.test_child + child_task = child.task_child + + flexmock(child_m) + .new_instances.should_receive(:can_be_deployed_by?) + .with(->(proxy) { proxy.__getobj__ == child_task }) + .and_return(false) + new_parent = syskit_deploy(parent_m) + new_child = new_parent.test_child + new_child_task = new_child.task_child + + assert_equal new_parent, parent + refute_equal new_child, child + refute_equal new_child_task, child_task + # And the old tasks should be ready to garbage-collect + assert_equal [child, child_task].to_set, + execute { plan.static_garbage_collect.to_set } + end + end + describe "#find_current_deployed_task" do it "ignores garbage tasks that have not been finalized yet" do component_m = Syskit::Component.new_submodel From 39586640206bf33765fb1c5798616c132fc0847a Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 11 Apr 2023 15:16:24 -0300 Subject: [PATCH 052/260] chore: style --- test/test_task_context.rb | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/test/test_task_context.rb b/test/test_task_context.rb index e87705a13..c3d3b9eba 100644 --- a/test/test_task_context.rb +++ b/test/test_task_context.rb @@ -2244,52 +2244,42 @@ def assert_process_events_does_not_block end describe "read_only" do + attr_reader :task, :task_m before do - task_m = TaskContext.new_submodel do + @task_m = TaskContext.new_submodel do property "p", "/double" end - @task = - syskit_stub_deploy_and_configure(task_m.with_arguments(read_only: true)) - + @task = syskit_stub_deploy_and_configure( + task_m.with_arguments(read_only: true) + ) Orocos.allow_blocking_calls { @task.orocos_task.configure(false) } end it "raises when attempting to change a property" do - task = @task - assert_raises(InvalidReadOnlyOperation) do task.properties.p = 2.0 end end it "emits start when the task is started while the component is running" do - task = @task Orocos.allow_blocking_calls { task.orocos_task.start(false) } - assert state(task.orocos_task) == :RUNNING - expect_execution { task.start! } - .to { emit task.start_event } + expect_execution { task.start! }.to { emit task.start_event } end - it "does not emit start " \ - "if the task is started while the component is not running" do - task = @task - + it "does not emit start if the task is started "\ + "while the component is not running" do assert state(task.orocos_task) != :RUNNING - expect_execution { task.start! } - .to { not_emit task.start_event } - + expect_execution { task.start! }.to { not_emit task.start_event } # just to avoid: "TeardownFailedError: failed to tear down plan" execute { task.stop! } end it "emits start when the component is started while the task is starting" do - task = @task execute { task.start! } - assert task.starting? expect_execution do @@ -2298,19 +2288,15 @@ def assert_process_events_does_not_block end it "does not emit interrupted " \ - "if the task is stopped while the component is running" do - task = @task + "if the task is stopped while the component is running" do Orocos.allow_blocking_calls { task.orocos_task.start(false) } execute { task.start! } assert task.running? && state(task.orocos_task) == :RUNNING - - expect_execution { task.stop! } - .to { not_emit task.interrupt_event } + expect_execution { task.stop! }.to { not_emit task.interrupt_event } end it "emits stop when the component is stopped while the task is running" do - task = @task Orocos.allow_blocking_calls { task.orocos_task.start(false) } execute { task.start! } From 878ed0482280cafe89dfb2900f442fff04190010 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 11 Apr 2023 15:16:40 -0300 Subject: [PATCH 053/260] fix: read only tasks not getting the message that the component is already in running state Simply refactoring the existing tests was enough, but it was mostly triggered when restarting a read-only task. The state reader would be "empty", that is the transition to RUNNING was already read, and therefore the state update poll would never trigger the start event. We now initialize a task's orogen state with the last known state of the component if available. --- lib/syskit/task_context.rb | 7 +- test/test_task_context.rb | 172 +++++++++++++++++++++++++++---------- 2 files changed, 134 insertions(+), 45 deletions(-) diff --git a/lib/syskit/task_context.rb b/lib/syskit/task_context.rb index a84aead2d..64fa8412e 100644 --- a/lib/syskit/task_context.rb +++ b/lib/syskit/task_context.rb @@ -560,7 +560,12 @@ def validate_orogen_state_from_rtt_state # task using the values read from the state reader def update_orogen_state @state_sample ||= state_reader.new_sample - state = state_reader.read_new(@state_sample) + state = + if @orogen_state + state_reader.read_new(@state_sample) + else + state_reader.read(@state_sample) + end if @exception_transition_deadline return update_orogen_state_in_exception(state) diff --git a/test/test_task_context.rb b/test/test_task_context.rb index c3d3b9eba..d1c4d7ee2 100644 --- a/test/test_task_context.rb +++ b/test/test_task_context.rb @@ -538,49 +538,81 @@ def start_task @task = flexmock(task) syskit_start_execution_agents(task) @orocos_task = flexmock(task.orocos_task) + setup_task_state_queue(task) + end + + def setup_task_state_queue(task) flexmock(task.state_reader) + @current_state = nil + @state_queue = [] + task.state_reader.should_receive(:read) + .at_most.once + .by_default + .and_return do + if @state_queue.empty? + @current_state + else + @current_state = @state_queue.shift + end + end + task.state_reader.should_receive(:read_new) + .by_default + .and_return { @current_state = @state_queue.shift } + end + + def push_task_state(state) + @state_queue << state end # NOTE: handling of errors related to the state readers is done # in live/test_state_reader_disconnection.rb it "is provided a connected state reader by its execution agent" do - syskit_start_execution_agents(task) assert task.state_reader.connected? end - it "sets orogen_state with the new state" do + it "reads the last known state on initialization "\ + "if there is no state transition" do + task.state_reader.should_receive(:read_new).and_return(nil) + task.state_reader.should_receive(:read).and_return(state = Object.new) + assert_equal state, task.update_orogen_state + refute task.last_orogen_state + end + it "does not read the last known state once it is initialized" do + task.state_reader.should_receive(:read) + .once.and_return(state = Object.new) task.state_reader.should_receive(:read_new) - .and_return(state = Object.new) + .once.and_return(nil) + assert_equal state, task.update_orogen_state + assert_nil task.update_orogen_state + end + it "sets orogen_state with the new state" do + push_task_state(state = Object.new) task.update_orogen_state assert_equal state, task.orogen_state end it "updates last_orogen_state with the current state" do - task.state_reader.should_receive(:read_new) - .and_return(last_state = Object.new) - .and_return(Object.new) + push_task_state(last_state = Object.new) + push_task_state(Object.new) task.update_orogen_state task.update_orogen_state assert_equal last_state, task.last_orogen_state end it "returns nil if no new state has been received" do - task.state_reader.should_receive(:read_new) - assert !task.update_orogen_state + refute task.update_orogen_state end it "does not change the last and current states if no new states "\ "have been received" do - task.state_reader.should_receive(:read_new) - .and_return(last_state = Object.new) - .and_return(state = Object.new) - .and_return(nil) + push_task_state(last_state = Object.new) + push_task_state(state = Object.new) + push_task_state(nil) task.update_orogen_state task.update_orogen_state - assert !task.update_orogen_state + refute task.update_orogen_state assert_equal last_state, task.last_orogen_state assert_equal state, task.orogen_state end it "returns the new state if there is one" do - task.state_reader.should_receive(:read_new) - .and_return(state = Object.new) + push_task_state(state = Object.new) assert_equal state, task.update_orogen_state end it "emits the exception event when transitioned to exception" do @@ -2244,67 +2276,119 @@ def assert_process_events_does_not_block end describe "read_only" do - attr_reader :task, :task_m + attr_reader :deployment, :handle before do - @task_m = TaskContext.new_submodel do + task_m = TaskContext.new_submodel do property "p", "/double" end - @task = syskit_stub_deploy_and_configure( - task_m.with_arguments(read_only: true) + @deployment = syskit_stub_deployment( + "test", task_model: task_m, read_only: ["test"] ) - Orocos.allow_blocking_calls { @task.orocos_task.configure(false) } - end + expect_execution { deployment.start! }.to { emit deployment.ready_event } - it "raises when attempting to change a property" do - assert_raises(InvalidReadOnlyOperation) do - task.properties.p = 2.0 - end + @handle = deployment.remote_task_handles["test"].handle end - it "emits start when the task is started while the component is running" do - Orocos.allow_blocking_calls { task.orocos_task.start(false) } - assert state(task.orocos_task) == :RUNNING + it "emits start when the task is create and started "\ + "while the component is running" do + Orocos.allow_blocking_calls { handle.configure(false) } + Orocos.allow_blocking_calls { handle.start(false) } + assert state(handle) == :RUNNING + create_configure_and_start_task + end + + it "emits start when the task is created and started after the component "\ + "configuration but before its start" do + Orocos.allow_blocking_calls { handle.configure(false) } + task = create_and_configure_task + Orocos.allow_blocking_calls { handle.start(false) } expect_execution { task.start! }.to { emit task.start_event } end it "does not emit start if the task is started "\ "while the component is not running" do - assert state(task.orocos_task) != :RUNNING - + task = create_and_configure_task expect_execution { task.start! }.to { not_emit task.start_event } # just to avoid: "TeardownFailedError: failed to tear down plan" execute { task.stop! } end it "emits start when the component is started while the task is starting" do + task = create_and_configure_task execute { task.start! } assert task.starting? expect_execution do - Orocos.allow_blocking_calls { task.orocos_task.start(false) } + Orocos.allow_blocking_calls { handle.configure(false) } + Orocos.allow_blocking_calls { handle.start(false) } end.to { emit task.start_event } end - it "does not emit interrupted " \ - "if the task is stopped while the component is running" do - Orocos.allow_blocking_calls { task.orocos_task.start(false) } - execute { task.start! } + it "raises when attempting to change a property" do + Orocos.allow_blocking_calls { handle.configure(false) } + Orocos.allow_blocking_calls { handle.start(false) } + task = create_configure_and_start_task - assert task.running? && state(task.orocos_task) == :RUNNING - expect_execution { task.stop! }.to { not_emit task.interrupt_event } + assert_raises(InvalidReadOnlyOperation) do + task.properties.p = 2.0 + end end - it "emits stop when the component is stopped while the task is running" do - Orocos.allow_blocking_calls { task.orocos_task.start(false) } - execute { task.start! } + describe "stopping behavior" do + attr_reader :task + + before do + Orocos.allow_blocking_calls { handle.configure(false) } + Orocos.allow_blocking_calls { handle.start(false) } + @task = create_configure_and_start_task + end - assert task.running? + it "does not emit interrupted " \ + "if the task is stopped while the component is running" do + expect_execution { task.stop! }.to do + not_emit task.interrupt_event + emit task.stop_event + end + end - expect_execution do - Orocos.allow_blocking_calls { task.orocos_task.stop(false) } - end.to { emit task.stop_event } + it "does not stop the component if the task is stopped" do + expect_execution { task.stop! }.to { emit task.stop_event } + assert state(handle) == :RUNNING + end + + it "emits stop when the component is stopped while the task is running" do + expect_execution do + Orocos.allow_blocking_calls { handle.stop(false) } + end.to { emit task.stop_event } + end + end + + it "handles being restarted on the same running component" do + Orocos.allow_blocking_calls { handle.configure(false) } + Orocos.allow_blocking_calls { handle.start(false) } + + 2.times do + task = deployment.task("test") + syskit_configure(task) + expect_execution { task.start! }.to { emit task.start_event } + expect_execution { task.stop! }.to { emit task.stop_event } + end + end + + def create_and_configure_task + task = @deployment.task("test") + syskit_configure(task) + task + end + + def create_configure_and_start_task + task = create_and_configure_task + expect_execution { task.start! }.to { emit task.start_event } + + assert task.running? && state(handle) == :RUNNING + task end def state(component) From fad2491989b25d1e414935c3eba0207d94e22458 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 20 Apr 2023 17:29:40 -0300 Subject: [PATCH 054/260] fix: apply example arguments to atomic actions in profile assertions This allows to handle a lot more actions with arguments in bulk tests (e.g. assert_can_deploy) instead of having to pass them explicitly --- lib/syskit/test/profile_assertions.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/syskit/test/profile_assertions.rb b/lib/syskit/test/profile_assertions.rb index ae98d2d06..4070ac3b5 100644 --- a/lib/syskit/test/profile_assertions.rb +++ b/lib/syskit/test/profile_assertions.rb @@ -153,17 +153,18 @@ def AtomicActions(arg) def BulkAssertAtomicActions(arg, exclude: []) exclude = ActionModels(exclude) skipped_actions = [] - actions = AtomicActions(arg).find_all do |action| + actions = AtomicActions(arg).map do |action| + action = action.dup.with_example_arguments if exclude.include?(action.model) - false + nil elsif !action.kind_of?(Actions::Action) && action.has_missing_required_arg? skipped_actions << action - false + nil else - true + action end - end + end.compact skipped_actions.delete_if do |skipped_action| actions.any? { |action| action.model == skipped_action.model } end From 4c99b90bf210d115e0ab424741391b7ace1429d5 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 26 Apr 2023 10:28:52 -0300 Subject: [PATCH 055/260] fix: change setup for read only tasks Instead of always considering that a read_only task setup is done, we will consider that it is done whenever its not-read only counterpart is running. Also, read_only tasks do nothing on perform_setup. --- lib/syskit/task_context.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/syskit/task_context.rb b/lib/syskit/task_context.rb index 64fa8412e..e3c0100bf 100644 --- a/lib/syskit/task_context.rb +++ b/lib/syskit/task_context.rb @@ -672,6 +672,8 @@ def read_current_state # Returns true if this component needs to be setup by calling the # #setup method, or if it can be used as-is def ready_for_setup?(state = nil) + return running? if read_only? + if execution_agent.configuring?(orocos_name) debug { "#{self} not ready for setup: already configuring" } return false @@ -718,7 +720,7 @@ def ready_for_setup?(state = nil) # end # def setup? - read_only? || @setup + @setup end def ready_to_start! @@ -921,6 +923,8 @@ def prepare_for_setup(promise) # (see Component#perform_setup) def perform_setup(promise) + return if read_only? + prepare_for_setup(promise) # This calls #configure From 7e0654c00dc12d88ef68a78a569c618e9f4d3463 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 26 Apr 2023 13:51:24 -0300 Subject: [PATCH 056/260] fix: ready for setup if orocos task is in runtime state when read_only --- lib/syskit/task_context.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/syskit/task_context.rb b/lib/syskit/task_context.rb index e3c0100bf..364d4f205 100644 --- a/lib/syskit/task_context.rb +++ b/lib/syskit/task_context.rb @@ -672,8 +672,6 @@ def read_current_state # Returns true if this component needs to be setup by calling the # #setup method, or if it can be used as-is def ready_for_setup?(state = nil) - return running? if read_only? - if execution_agent.configuring?(orocos_name) debug { "#{self} not ready for setup: already configuring" } return false @@ -696,6 +694,8 @@ def ready_for_setup?(state = nil) return end + return orocos_task.runtime_state?(state) if read_only? + configurable_state = CONFIGURABLE_RTT_STATES.include?(state) || orocos_task.exception_state?(state) From 0db941c51213c7f511e1d9af9e80945d63f36fb5 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 26 Apr 2023 13:52:00 -0300 Subject: [PATCH 057/260] fix(test): do not configure read only tasks --- lib/syskit/test/network_manipulation.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/syskit/test/network_manipulation.rb b/lib/syskit/test/network_manipulation.rb index f4ad9329e..c29a2f1af 100644 --- a/lib/syskit/test/network_manipulation.rb +++ b/lib/syskit/test/network_manipulation.rb @@ -545,6 +545,8 @@ def syskit_configure( current_pending = pending.size has_missing_states = false pending.delete_if do |t| + next true if t.read_only? + t.freeze_delayed_arguments should_setup = Orocos.allow_blocking_calls do if !t.kind_of?(Syskit::TaskContext) From 372755a5771284b9cc89164d50452396565f9fa9 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 26 Apr 2023 13:52:20 -0300 Subject: [PATCH 058/260] feat(test): add tests for read only tasks on #ready_for_setup and #perform_setup --- test/test_task_context.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/test_task_context.rb b/test/test_task_context.rb index d1c4d7ee2..05cee02a2 100644 --- a/test/test_task_context.rb +++ b/test/test_task_context.rb @@ -700,6 +700,34 @@ def push_task_state(state) task.should_receive(:read_current_state).and_return(:STOPPED) assert task.ready_for_setup? end + + it "returns false if the task is read_only and the component is not running" do + task.should_receive(:read_only?).and_return(true) + refute task.ready_for_setup? + end + + it "returns true if the task is read_only and the component is in a running "\ + "state" do + task.should_receive(:read_only?).and_return(true) + orocos_task.should_receive(:runtime_state?) + .and_return(true) + assert task.ready_for_setup? + end + end + + describe "#perform_setup" do + attr_reader :task + before do + task = syskit_stub_and_deploy("Task") {} + syskit_start_execution_agents(task) + @task = flexmock(task) + end + + it "does nothing when the task is read_only" do + task.should_receive(:read_only?).and_return(true) + task.should_receive(:prepare_for_setup).never + flexmock(task.orocos_task).should_receive(:configure).never + end end describe "#read_current_state" do From 11fd6b9ae6864fe6309b1d5e006bd449e39b94d1 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 26 Apr 2023 16:22:30 -0300 Subject: [PATCH 059/260] fix: update read-only tests now that it needs the component to be started before configuration The change we just made was to require the component to be started for the read-only task to let itself be configured. --- lib/syskit/task_context.rb | 38 +++++++++++++++++-------- lib/syskit/test/network_manipulation.rb | 2 -- test/test_task_context.rb | 34 ++++++---------------- 3 files changed, 35 insertions(+), 39 deletions(-) diff --git a/lib/syskit/task_context.rb b/lib/syskit/task_context.rb index 364d4f205..829d44513 100644 --- a/lib/syskit/task_context.rb +++ b/lib/syskit/task_context.rb @@ -694,22 +694,36 @@ def ready_for_setup?(state = nil) return end - return orocos_task.runtime_state?(state) if read_only? - - configurable_state = - CONFIGURABLE_RTT_STATES.include?(state) || - orocos_task.exception_state?(state) - if configurable_state - true + if read_only? + read_only_task_verify_configurable_state(state) else - debug do - "#{self} not ready for setup: in state #{state}, "\ - "expected STOPPED, PRE_OPERATIONAL or an exception state" - end - false + normal_task_verify_configurable_state(state) end end + def read_only_task_verify_configurable_state(state) + return true if orocos_task.runtime_state?(state) + + execution_engine.scheduler.report_holdoff( + "read-only task #{self} not ready, task is not currently running", self + ) + false + end + + def normal_task_verify_configurable_state(state) + configurable = + CONFIGURABLE_RTT_STATES.include?(state) || + orocos_task.exception_state?(state) + + return true if configurable + + execution_engine.scheduler.report_holdoff( + "task #{self} not ready, task is in state #{state}, expected "\ + "PRE_OPERATIONAL, STOPPED or an exception state", self + ) + false + end + # Returns true if the underlying Orocos task has been configured and # can be started # diff --git a/lib/syskit/test/network_manipulation.rb b/lib/syskit/test/network_manipulation.rb index c29a2f1af..f4ad9329e 100644 --- a/lib/syskit/test/network_manipulation.rb +++ b/lib/syskit/test/network_manipulation.rb @@ -545,8 +545,6 @@ def syskit_configure( current_pending = pending.size has_missing_states = false pending.delete_if do |t| - next true if t.read_only? - t.freeze_delayed_arguments should_setup = Orocos.allow_blocking_calls do if !t.kind_of?(Syskit::TaskContext) diff --git a/test/test_task_context.rb b/test/test_task_context.rb index 05cee02a2..0718553fb 100644 --- a/test/test_task_context.rb +++ b/test/test_task_context.rb @@ -2322,36 +2322,20 @@ def assert_process_events_does_not_block "while the component is running" do Orocos.allow_blocking_calls { handle.configure(false) } Orocos.allow_blocking_calls { handle.start(false) } - assert state(handle) == :RUNNING - create_configure_and_start_task end - it "emits start when the task is created and started after the component "\ - "configuration but before its start" do - Orocos.allow_blocking_calls { handle.configure(false) } - task = create_and_configure_task - Orocos.allow_blocking_calls { handle.start(false) } - expect_execution { task.start! }.to { emit task.start_event } - end - - it "does not emit start if the task is started "\ - "while the component is not running" do - task = create_and_configure_task - expect_execution { task.start! }.to { not_emit task.start_event } - # just to avoid: "TeardownFailedError: failed to tear down plan" - execute { task.stop! } + it "does not let itself be configured if the component is not configured" do + assert_raises(Syskit::Test::NetworkManipulation::NoConfigureFixedPoint) do + create_and_configure_task + end end - it "emits start when the component is started while the task is starting" do - task = create_and_configure_task - execute { task.start! } - assert task.starting? - - expect_execution do - Orocos.allow_blocking_calls { handle.configure(false) } - Orocos.allow_blocking_calls { handle.start(false) } - end.to { emit task.start_event } + it "does not let itself be configured if the component is not started" do + Orocos.allow_blocking_calls { handle.configure(false) } + assert_raises(Syskit::Test::NetworkManipulation::NoConfigureFixedPoint) do + create_and_configure_task + end end it "raises when attempting to change a property" do From 29136ff89fee92d5a96824af90b06136b49b85e9 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 26 Apr 2023 17:11:56 -0300 Subject: [PATCH 060/260] fix(test): try to start read only task on perform_setup test --- test/test_task_context.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_task_context.rb b/test/test_task_context.rb index 0718553fb..6c5616a00 100644 --- a/test/test_task_context.rb +++ b/test/test_task_context.rb @@ -727,6 +727,9 @@ def push_task_state(state) task.should_receive(:read_only?).and_return(true) task.should_receive(:prepare_for_setup).never flexmock(task.orocos_task).should_receive(:configure).never + expect_execution { task.start! }.to do + not_emit task.start_event, within: 0.5 + end end end From bf69de740751f6f12ca65b8e005b7a4fc3052423 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 26 Apr 2023 18:18:02 -0300 Subject: [PATCH 061/260] fix(test): check if read_only tasks properly ignore perform_setup --- lib/syskit/task_context.rb | 2 +- test/test_task_context.rb | 30 +++++++++++------------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/lib/syskit/task_context.rb b/lib/syskit/task_context.rb index 829d44513..c6fb4a785 100644 --- a/lib/syskit/task_context.rb +++ b/lib/syskit/task_context.rb @@ -713,7 +713,7 @@ def read_only_task_verify_configurable_state(state) def normal_task_verify_configurable_state(state) configurable = CONFIGURABLE_RTT_STATES.include?(state) || - orocos_task.exception_state?(state) + orocos_task.exception_state?(state) return true if configurable diff --git a/test/test_task_context.rb b/test/test_task_context.rb index 6c5616a00..41affb835 100644 --- a/test/test_task_context.rb +++ b/test/test_task_context.rb @@ -710,29 +710,11 @@ def push_task_state(state) "state" do task.should_receive(:read_only?).and_return(true) orocos_task.should_receive(:runtime_state?) - .and_return(true) + .and_return(true) assert task.ready_for_setup? end end - describe "#perform_setup" do - attr_reader :task - before do - task = syskit_stub_and_deploy("Task") {} - syskit_start_execution_agents(task) - @task = flexmock(task) - end - - it "does nothing when the task is read_only" do - task.should_receive(:read_only?).and_return(true) - task.should_receive(:prepare_for_setup).never - flexmock(task.orocos_task).should_receive(:configure).never - expect_execution { task.start! }.to do - not_emit task.start_event, within: 0.5 - end - end - end - describe "#read_current_state" do attr_reader :task, :state_reader before do @@ -2351,6 +2333,16 @@ def assert_process_events_does_not_block end end + it "does not perform setup on configuration" do + task = deployment.task("test") + flexmock(task).should_receive(:perform_setup).once + flexmock(task).should_receive(:prepare_for_setup).never + + Orocos.allow_blocking_calls { handle.configure(false) } + Orocos.allow_blocking_calls { handle.start(false) } + syskit_configure(task) + end + describe "stopping behavior" do attr_reader :task From 9b4b666d87a38822463b638f52c9a3524975403e Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sat, 25 Mar 2023 21:09:03 -0300 Subject: [PATCH 062/260] fix(cli): make `help orogen-test` work orogen-test is special because it is represented internally as orogen_test by thor. Some codepaths of bin/syskit were handling that, but the one that deals with `help` was not. --- bin/syskit | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bin/syskit b/bin/syskit index 349e87338..3aa12de1f 100755 --- a/bin/syskit +++ b/bin/syskit @@ -17,20 +17,23 @@ end require "syskit/cli/main" +def thor_command?(name) + Syskit::CLI::Main.all_commands.key?(name.tr("-", "_")) +end + # Transform 'syskit help ' to 'syskit --help' if mode is not # handled by thor if ARGV[0] == "help" - if (command_name = ARGV[1]) - unless Syskit::CLI::Main.all_commands.key?(command_name) - ARGV.shift - ARGV << "--help" - end + if ((command_name = ARGV[1]) && !thor_command?(command_name)) + ARGV.shift + ARGV << "--help" end # Transform 'syskit --help' to 'syskit help ' if mode is handled # by thor elsif ["-h", "--help"].include?(ARGV[1]) STDERR.puts "WARN: syskit --help is deprecated, use syskit help instead" - if Syskit::CLI::Main.all_commands.key?(ARGV[0]) + if thor_command?(ARGV[0]) + ARGV.delete_at(1) ARGV.unshift "help" end end From 904c9b09645dbce4682bb6812cb3f204428be755 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sat, 25 Mar 2023 21:09:21 -0300 Subject: [PATCH 063/260] fix: forward --log from orogen-test to `roby test` --- lib/syskit/cli/main.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/syskit/cli/main.rb b/lib/syskit/cli/main.rb index aadb4213d..043223e65 100644 --- a/lib/syskit/cli/main.rb +++ b/lib/syskit/cli/main.rb @@ -18,6 +18,7 @@ class Main < Roby::CLI::Main option :workdir, type: :string, default: nil option :logs, type: :string, default: nil option :logs_base, type: :string, default: nil + option :log, type: :string, repeatable: true, default: [] def orogen_test(*args) syskit_path = File.expand_path("../../../bin/syskit", __dir__) minitest_args, files = args.partition { |p| p.start_with?("-") } @@ -28,6 +29,7 @@ def orogen_test(*args) extra_args = ["--keep-logs"] extra_args << "--logs" << options[:logs] if options[:logs] extra_args << "--logs-base" << options[:logs_base] if options[:logs_base] + extra_args.concat(options[:log].map { |l| "--log=#{l}" }) system(syskit_path, "gen", "app", workdir) unless File.directory?(workdir) Process.exec(syskit_path, "test", "--live", *extra_args, *files, "--", From 1fec9bac1d94db63e918801857d1bf4344342ff3 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Fri, 28 Apr 2023 11:09:10 -0300 Subject: [PATCH 064/260] feat: show rejected samples when have_new_samples().matching { } fails --- .rubocop_todo.yml | 1 - lib/syskit/test/execution_expectations.rb | 37 ++++++++++++++++++++--- test/test/test_execution_expectations.rb | 22 ++++++++++++-- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 470141000..b5f6e3277 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -259,7 +259,6 @@ Lint/AssignmentInCondition: - 'lib/syskit/runtime/connection_management.rb' - 'lib/syskit/scripts/common.rb' - 'lib/syskit/task_configuration_manager.rb' - - 'test/test/test_execution_expectations.rb' - 'test/test_remote_state_getter.rb' # Offense count: 23 diff --git a/lib/syskit/test/execution_expectations.rb b/lib/syskit/test/execution_expectations.rb index 147c67824..d5b645ec6 100644 --- a/lib/syskit/test/execution_expectations.rb +++ b/lib/syskit/test/execution_expectations.rb @@ -129,17 +129,14 @@ class HaveNewSamples < Roby::Test::ExecutionExpectations::Achieve def initialize(reader, count, buffer_size, backtrace) @received_samples = [] + @rejected_samples = [] @predicate = nil orocos_reader = ExecutionExpectations.resolve_orocos_reader( reader, type: :buffer, size: buffer_size ) - description = proc do - matching = " matching the given predicate" if @predicate - "#{reader} should have received #{count} new sample(s)"\ - "#{matching}, but got #{@received_samples.size}" - end + description = proc { format_error_message(reader, count) } block = ->(_) { process_samples(orocos_reader, count) } super(block, description, backtrace) end @@ -149,6 +146,8 @@ def process_samples(reader, count) if !@predicate || @predicate.call(sample) @received_samples << sample return true if @received_samples.size == count + else + @rejected_samples << sample end end false @@ -158,6 +157,34 @@ def return_object @received_samples end + def format_error_message(reader, expected_count) + matching = " matching the given predicate" if @predicate + msg = "#{reader} should have received #{expected_count} "\ + "new sample(s)#{matching}, but got #{@received_samples.size}" + + return msg unless @predicate + + msg + "\n " + format_rejected_samples_message + end + + def format_rejected_samples_message + if @rejected_samples.empty? + return "No samples were rejected by the #matching predicate" + elsif @rejected_samples.size > 10 + samples = @rejected_samples[-10, 10] + else + samples = @rejected_samples + end + + msg = +"#{samples.size} samples were rejected "\ + "by the #matching predicate:" + samples.each do |s| + sample_to_s = PP.pp(s, +"") + msg += "\n #{sample_to_s.split("\n").join("\n ")}" + end + msg + end + def matching(&block) if @predicate raise ArgumentError, "only one #matching predicate is allowed" diff --git a/test/test/test_execution_expectations.rb b/test/test/test_execution_expectations.rb index 2ff92050b..32115aaf5 100644 --- a/test/test/test_execution_expectations.rb +++ b/test/test/test_execution_expectations.rb @@ -250,15 +250,31 @@ module Test ).matching { |v| v >= default_size } end end + it "produces a specific message if there were no rejected samples" do + e = assert_raises(Roby::Test::ExecutionExpectations::Unmet) do + expect_execution { syskit_write task.in_port, 2 } + .timeout(0.01) + .to { have_new_samples(task.in_port, 2).matching(&:even?) } + end + expected_msg = <<~MSG.chomp + #{task.in_port} should have received 2 new sample(s) matching the given predicate, but got 1 + No samples were rejected by the #matching predicate + MSG + assert_equal expected_msg, e.message.split("\n")[1, 4].join("\n") + end it "fails if the task does not emit enough matching samples" do e = assert_raises(Roby::Test::ExecutionExpectations::Unmet) do expect_execution { syskit_write task.in_port, 1, 2, 3 } .timeout(0.01) .to { have_new_samples(task.in_port, 2).matching(&:even?) } end - assert_match "#{task.in_port} should have received 2 new "\ - "sample(s) matching the given predicate, "\ - "but got 1", e.message.split("\n")[1] + expected_msg = <<~MSG.chomp + #{task.in_port} should have received 2 new sample(s) matching the given predicate, but got 1 + 2 samples were rejected by the #matching predicate: + 1 + 3 + MSG + assert_equal expected_msg, e.message.split("\n")[1, 4].join("\n") end it "provides the backtrace from the point of call by default" do expectation = nil From 9930966e4f08314445bac151adefcf314cce3e21 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Fri, 28 Apr 2023 17:52:49 -0300 Subject: [PATCH 065/260] fix(test): add pass thru to perform setup mock --- test/test_task_context.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_task_context.rb b/test/test_task_context.rb index 41affb835..6dce68a76 100644 --- a/test/test_task_context.rb +++ b/test/test_task_context.rb @@ -2335,7 +2335,7 @@ def assert_process_events_does_not_block it "does not perform setup on configuration" do task = deployment.task("test") - flexmock(task).should_receive(:perform_setup).once + flexmock(task).should_receive(:perform_setup).once.pass_thru flexmock(task).should_receive(:prepare_for_setup).never Orocos.allow_blocking_calls { handle.configure(false) } From fa90e040eaa39a46111d83760176c4cbaad2428f Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 30 Apr 2023 17:13:59 -0300 Subject: [PATCH 066/260] fix: mention that we are showing only the last 10 messages in .match failure --- lib/syskit/test/execution_expectations.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/syskit/test/execution_expectations.rb b/lib/syskit/test/execution_expectations.rb index d5b645ec6..54f5951b8 100644 --- a/lib/syskit/test/execution_expectations.rb +++ b/lib/syskit/test/execution_expectations.rb @@ -170,14 +170,18 @@ def format_error_message(reader, expected_count) def format_rejected_samples_message if @rejected_samples.empty? return "No samples were rejected by the #matching predicate" - elsif @rejected_samples.size > 10 + end + + if @rejected_samples.size > 10 samples = @rejected_samples[-10, 10] + extra_msg = ". The last 10 rejected samples were:" else samples = @rejected_samples + extra_msg = ":" end msg = +"#{samples.size} samples were rejected "\ - "by the #matching predicate:" + "by the #matching predicate#{extra_msg}" samples.each do |s| sample_to_s = PP.pp(s, +"") msg += "\n #{sample_to_s.split("\n").join("\n ")}" From 6a69e4df0461a3a936f35b2dea714e10ee2ddd18 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sat, 3 Jun 2023 18:16:32 -0300 Subject: [PATCH 067/260] fix: do not terminate early dataflow computations when a port is not updated The logic was broken in add_port_info, which would reset @changed to false if the particular port was not updated, leading to the propagation algorithm to think it found a fixed point. --- .../dataflow_computation.rb | 16 ++++++- .../test_dataflow_computation.rb | 46 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 test/network_generation/test_dataflow_computation.rb diff --git a/lib/syskit/network_generation/dataflow_computation.rb b/lib/syskit/network_generation/dataflow_computation.rb index 3af7909dd..a964a5c09 100644 --- a/lib/syskit/network_generation/dataflow_computation.rb +++ b/lib/syskit/network_generation/dataflow_computation.rb @@ -154,6 +154,20 @@ def pretty_print(pp) end end + # @api private + # + # For testing purposes only + def changed? + @changed + end + + # @api private + # + # For testing purposes only + def reset_changed + @changed = false + end + def reset(tasks = []) @result = Hash.new { |h, k| h[k] = {} } # Internal variable that is used to detect whether an iteration @@ -318,7 +332,7 @@ def add_port_info(task, port_name, info) @result[task][port_name] = info else begin - @changed = @result[task][port_name].merge(info) + @changed |= @result[task][port_name].merge(info) rescue Exception => e raise e, "while adding information to port #{port_name} on #{task}, #{e.message}", e.backtrace end diff --git a/test/network_generation/test_dataflow_computation.rb b/test/network_generation/test_dataflow_computation.rb new file mode 100644 index 000000000..d867ddbb4 --- /dev/null +++ b/test/network_generation/test_dataflow_computation.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "syskit/test/self" + +module Syskit + module NetworkGeneration + describe DataFlowComputation do + describe "#add_port_info" do + before do + @computation = DataFlowComputation.new + end + + it "sets the port info to the new information object if it did not have "\ + "any" do + @computation.add_port_info(@task, "port", info = flexmock) + assert_equal info, @computation.port_info(@task, "port") + end + + it "calls #merge on the existing port info when it gets updated" do + @computation.add_port_info(@task, "port", info = flexmock) + info.should_receive(:merge).with(new_info = flexmock).once + @computation.add_port_info(@task, "port", new_info) + end + + it "sets changed? if a port that had no previous info gets some" do + @computation.add_port_info(@task, "port", flexmock) + assert @computation.changed? + end + + it "sets changed? if a port with existing information is updated" do + @computation.add_port_info(@task, "port", flexmock(merge: true)) + @computation.reset_changed + refute @computation.changed? + @computation.add_port_info(@task, "port", flexmock) + assert @computation.changed? + end + + it "keeps changed? set even if the port is not updated" do + @computation.add_port_info(@task, "port", flexmock(merge: false)) + @computation.add_port_info(@task, "port", flexmock) + assert @computation.changed? + end + end + end + end +end From d3887c51ad3350f93c543f6f3cc11c89d7c9bf98 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 2 May 2023 17:10:38 -0300 Subject: [PATCH 068/260] fix: add test for BulkAssertAtomicActions using example as default arguments --- test/test/test_profile_assertions.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test/test_profile_assertions.rb b/test/test/test_profile_assertions.rb index bac57468a..c2c89f72d 100644 --- a/test/test/test_profile_assertions.rb +++ b/test/test/test_profile_assertions.rb @@ -232,6 +232,19 @@ module Test assert_equal [], found assert_equal [], skipped end + + it "uses example arguments in place of required arguments "\ + "when available" do + @interface_m.describe(:m_action).required_arg(:test, "", example: 10) + @interface_m.define_method :m_action do |test:| + end + + found, skipped = BulkAssertAtomicActions( + @interface_m.m_action + ) + assert_equal [@interface_m.m_action.with_arguments(test: 10)], found + assert_equal [], skipped + end end describe "assert_is_self_contained" do From 934efda22fca4851b54150958bec99d4d342b332 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 16 May 2023 23:03:46 -0300 Subject: [PATCH 069/260] chore: small simplification --- lib/syskit/test/network_manipulation.rb | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/lib/syskit/test/network_manipulation.rb b/lib/syskit/test/network_manipulation.rb index f4ad9329e..e57b27a98 100644 --- a/lib/syskit/test/network_manipulation.rb +++ b/lib/syskit/test/network_manipulation.rb @@ -200,7 +200,25 @@ def syskit_deploy( end.compact root_tasks = placeholder_tasks.map(&:as_service) placeholder_tasks = normalize_instanciation_models(placeholder_tasks) + syskit_deploy_normalized_placeholder_tasks( + placeholder_tasks, + syskit_engine: syskit_engine, + default_deployment_group: default_deployment_group, + **resolve_options + ) + + root_tasks = root_tasks.map(&:task) + if root_tasks.size == 1 + root_tasks.first + elsif root_tasks.size > 1 + root_tasks + end + end + def syskit_deploy_normalized_placeholder_tasks( + placeholder_tasks, + syskit_engine:, default_deployment_group:, **resolve_options + ) requirement_tasks = placeholder_tasks.map(&:planning_task) not_running = requirement_tasks.find_all { |t| !t.running? } @@ -234,13 +252,6 @@ def syskit_deploy( end requirement_tasks.each { |t| t.success_event.emit unless t.finished? } end - - root_tasks = root_tasks.map(&:task) - if root_tasks.size == 1 - root_tasks.first - elsif root_tasks.size > 1 - root_tasks - end end def syskit_engine_resolve_handle_plan_export From 2b2c30cc3b3d1161fce88ad78dbb3a051aa4e0fa Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 29 May 2023 19:14:30 -0300 Subject: [PATCH 070/260] fix: recursively expand coordination models in profile assertions --- lib/syskit/test/profile_assertions.rb | 80 ++++++++++++++++++++------- test/test/test_profile_assertions.rb | 20 +++++++ 2 files changed, 79 insertions(+), 21 deletions(-) diff --git a/lib/syskit/test/profile_assertions.rb b/lib/syskit/test/profile_assertions.rb index 4070ac3b5..73ae4c880 100644 --- a/lib/syskit/test/profile_assertions.rb +++ b/lib/syskit/test/profile_assertions.rb @@ -138,9 +138,20 @@ def Actions(arg) # Like #Actions, but expands coordination models into their # consistuent actions def AtomicActions(arg) - Actions(arg).flat_map do |action| - expand_action_coordination_models(action) + queue = Actions(arg) + result = [] + until queue.empty? + actions = Actions(queue.shift).flat_map do |action| + expand_action_coordination_models(action) + end + + if actions.size == 1 + result.concat(actions) + else + queue.concat(actions) + end end + result.map { |a| a.dup.with_example_arguments } end # Like {#AtomicActions} but filters out actions that cannot be @@ -154,7 +165,6 @@ def BulkAssertAtomicActions(arg, exclude: []) exclude = ActionModels(exclude) skipped_actions = [] actions = AtomicActions(arg).map do |action| - action = action.dup.with_example_arguments if exclude.include?(action.model) nil elsif !action.kind_of?(Actions::Action) && @@ -314,10 +324,7 @@ def assert_can_instanciate( end actions.each do |action| - tasks = assert_can_instanciate_together(action, *together_with) - Array(tasks).each { |t| plan.unmark_mission_task(t) } - yield(action, tasks, together_with: together_with) if block_given? - expect_execution.garbage_collect(true).to_run + assert_can_instanciate_together(action, *together_with) end end @@ -341,11 +348,9 @@ def can_instanciate( # deployed (e.g. if some components do not have a corresponding # deployment) def assert_can_instanciate_together(*actions) - actions = subject_syskit_model if actions.empty? - self.assertions += 1 - syskit_deploy(AtomicActions(actions), - compute_policies: false, - compute_deployments: false) + syskit_run_deploy_in_bulk( + actions, compute_policies: false, compute_deployments: false + ) rescue Minitest::Assertion, StandardError => e raise ProfileAssertionFailed.new(actions, e), e.message, e.backtrace end @@ -430,10 +435,7 @@ def assert_can_deploy( end actions.each do |action| - task = assert_can_deploy_together(action, *together_with) - yield(action, tasks, together_with: together_with) if block_given? - Array(task).each { |t| plan.unmark_mission_task(t) } - expect_execution.garbage_collect(true).to_run + assert_can_deploy_together(action, *together_with) end end @@ -452,15 +454,51 @@ def can_deploy(action_or_profile = subject_syskit_model, together_with: []) # It is stronger (and therefore includes) # {assert_can_instanciate_together} def assert_can_deploy_together(*actions) - actions = subject_syskit_model if actions.empty? - self.assertions += 1 - syskit_deploy(AtomicActions(actions), - compute_policies: true, - compute_deployments: true) + syskit_run_deploy_in_bulk( + actions, compute_policies: true, compute_deployments: true + ) rescue Minitest::Assertion, StandardError => e raise ProfileAssertionFailed.new(actions, e), e.message, e.backtrace end + def syskit_run_deploy_in_bulk( + actions, compute_policies:, compute_deployments: + ) + actions = subject_syskit_model if actions.empty? + self.assertions += 1 + atomic_actions = actions.map { AtomicActions(_1) } + ProfileAssertions.each_combination(*atomic_actions) do |test_actions| + t = syskit_deploy( + *test_actions.map(&:with_example_arguments), + compute_policies: compute_policies, + compute_deployments: compute_deployments + ) + Array(t).each { plan.unmark_mission_task(_1) } + expect_execution.garbage_collect(true).to_run + end + end + + def self.each_combination(*arrays) + return enum_for(__method__, *arrays) unless block_given? + + enumerators = arrays.map(&:each) + i = 0 + values = [] + loop do + values[i] = enumerators[i].next + if i == enumerators.size - 1 + yield values.dup + else + i += 1 + end + rescue StopIteration + return if i == 0 + + enumerators[i] = arrays[i].each + i -= 1 + end + end + # Spec-style call for {#assert_can_deploy_together} # # @example diff --git a/test/test/test_profile_assertions.rb b/test/test/test_profile_assertions.rb index c2c89f72d..56d7b1e66 100644 --- a/test/test/test_profile_assertions.rb +++ b/test/test/test_profile_assertions.rb @@ -573,6 +573,26 @@ module Test assert_equal message, e.message end end + + describe ".each_combination" do + it "calculates and yields each possible combination of its arguments" do + result = ProfileAssertions.each_combination( + [1, 2, 3], + [4, 5], + [6, 7, 8] + ).to_a + + expected = [ + [1, 4, 6], [1, 4, 7], [1, 4, 8], + [1, 5, 6], [1, 5, 7], [1, 5, 8], + [2, 4, 6], [2, 4, 7], [2, 4, 8], + [2, 5, 6], [2, 5, 7], [2, 5, 8], + [3, 4, 6], [3, 4, 7], [3, 4, 8], + [3, 5, 6], [3, 5, 7], [3, 5, 8] + ] + assert_equal expected, result + end + end end end end From 8a0e2b6559b2e0bfcc4a6f8057c2a0a8bf6a7dcd Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 5 Jun 2023 15:59:29 -0300 Subject: [PATCH 071/260] fix: update tests in relation to a change to Task#to_s in Roby --- test/models/test_composition_child.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/models/test_composition_child.rb b/test/models/test_composition_child.rb index 7302866cb..59171e4b4 100644 --- a/test/models/test_composition_child.rb +++ b/test/models/test_composition_child.rb @@ -161,7 +161,7 @@ @cmp_m.test_child.bind(cmp.test_child.other_srv) end assert_equal "cannot bind Syskit::Models::Placeholder to "\ - "#", + "#", e.message.gsub(/0x[0-9a-f]+/, "0x") end end @@ -188,10 +188,11 @@ e = assert_raises(ArgumentError) do @cmp_m.test_child.bind(task) end - assert_equal "cannot bind cmp_m.test_child["\ - "Syskit::Models::Placeholder] to task_m:: "\ - "it is not the child of any cmp_m composition", - e.message.gsub(/:0x.*:/, "::") + expected = "cannot bind cmp_m.test_child[Syskit::Models::"\ + "Placeholder] to task_m(conf: default(["\ + "\"default\"]), read_only: default(false)): it is not"\ + " the child of any cmp_m composition" + assert_equal expected, e.message.gsub(/id:\d+/, "id:X") end it "does not move up to parents when the role does not match" do plan.add(task = @task_m.new) @@ -201,9 +202,10 @@ @cmp_m.test_child.bind(task) end assert_equal "cannot bind cmp_m.test_child[Syskit::Models::"\ - "Placeholder] to task_m:: it is the child of one "\ - "or more cmp_m compositions, but not with the role 'test'", - e.message.gsub(/:0x.*:/, "::") + "Placeholder] to task_m(conf: default(["\ + "\"default\"]), read_only: default(false)): it is the child "\ + "of one or more cmp_m compositions, but not with the role "\ + "'test'", e.message.gsub(/id:\d+/, "id:X") end end end From d5108c03fab4daf57c25dbd9beddda17651fce59 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 6 Jun 2023 12:22:34 -0300 Subject: [PATCH 072/260] fix: in readers, properly handle 'false' as a valid value returned by read and read_new --- lib/syskit/dynamic_port_binding.rb | 4 ++-- test/test_dynamic_port_binding.rb | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/syskit/dynamic_port_binding.rb b/lib/syskit/dynamic_port_binding.rb index 35c4151e5..b503a28d9 100644 --- a/lib/syskit/dynamic_port_binding.rb +++ b/lib/syskit/dynamic_port_binding.rb @@ -209,7 +209,7 @@ def create_accessor(port) # @return [nil,Typelib::Type] nil if there is has never been any data, or if # there are no underlying port. A data sample otherwise. def read(sample = nil) - return unless (sample = @resolved_accessor&.read(sample)) + return if (sample = @resolved_accessor&.read(sample)).nil? @value_resolver.__resolve(sample) end @@ -221,7 +221,7 @@ def read(sample = nil) # @return [nil,Typelib::Type] nil if there is no new data or if there are # no underlying port. A data sample otherwise. def read_new(sample = nil) - return unless (sample = @resolved_accessor&.read_new(sample)) + return if (sample = @resolved_accessor&.read_new(sample)).nil? @value_resolver.__resolve(sample) end diff --git a/test/test_dynamic_port_binding.rb b/test/test_dynamic_port_binding.rb index 7c1ea018c..abfb4a871 100644 --- a/test/test_dynamic_port_binding.rb +++ b/test/test_dynamic_port_binding.rb @@ -573,6 +573,14 @@ def wait_until_connected(accessor) assert_equal 2, @reader.read_new end + it "handles 'false' as a valid value" do + @reader.attach_to_task(@task) + wait_until_connected @reader + flexmock(@reader.resolved_accessor) + .should_receive(:read_new).and_return(false) + assert_equal false, @reader.read_new + end + it "processes the samples through the given value resolver" do @value_resolver.should_receive(:__resolve).with(2).and_return(42) @reader.attach_to_task(@task) @@ -638,6 +646,14 @@ def wait_until_connected(source) assert_equal 2, @reader.read end + it "handles 'false' as a valid value" do + @reader.attach_to_task(@task) + wait_until_connected @reader + flexmock(@reader.resolved_accessor) + .should_receive(:read).and_return(false) + assert_equal false, @reader.read + end + it "processes the samples through the given value resolver" do @value_resolver.should_receive(:__resolve).with(2.0).and_return(42) @reader.attach_to_task(@task) From 10feccc488c886a3c5c1769819e3241cc34232e2 Mon Sep 17 00:00:00 2001 From: Sylvain Joyeux Date: Tue, 6 Jun 2023 15:36:10 -0300 Subject: [PATCH 073/260] Revert "fix: update tests in relation to a change to Task#to_s in Roby" --- test/models/test_composition_child.rb | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/models/test_composition_child.rb b/test/models/test_composition_child.rb index 59171e4b4..7302866cb 100644 --- a/test/models/test_composition_child.rb +++ b/test/models/test_composition_child.rb @@ -161,7 +161,7 @@ @cmp_m.test_child.bind(cmp.test_child.other_srv) end assert_equal "cannot bind Syskit::Models::Placeholder to "\ - "#", + "#", e.message.gsub(/0x[0-9a-f]+/, "0x") end end @@ -188,11 +188,10 @@ e = assert_raises(ArgumentError) do @cmp_m.test_child.bind(task) end - expected = "cannot bind cmp_m.test_child[Syskit::Models::"\ - "Placeholder] to task_m(conf: default(["\ - "\"default\"]), read_only: default(false)): it is not"\ - " the child of any cmp_m composition" - assert_equal expected, e.message.gsub(/id:\d+/, "id:X") + assert_equal "cannot bind cmp_m.test_child["\ + "Syskit::Models::Placeholder] to task_m:: "\ + "it is not the child of any cmp_m composition", + e.message.gsub(/:0x.*:/, "::") end it "does not move up to parents when the role does not match" do plan.add(task = @task_m.new) @@ -202,10 +201,9 @@ @cmp_m.test_child.bind(task) end assert_equal "cannot bind cmp_m.test_child[Syskit::Models::"\ - "Placeholder] to task_m(conf: default(["\ - "\"default\"]), read_only: default(false)): it is the child "\ - "of one or more cmp_m compositions, but not with the role "\ - "'test'", e.message.gsub(/id:\d+/, "id:X") + "Placeholder] to task_m:: it is the child of one "\ + "or more cmp_m compositions, but not with the role 'test'", + e.message.gsub(/:0x.*:/, "::") end end end From 8eb09870071163770234c86e6ab19c6f3015ce42 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 31 May 2023 11:43:05 -0300 Subject: [PATCH 074/260] chore: refactor MergeSolver#resolve_merge to exfiltrate diagnostic information Meant to be used to provide more useful information in case of failures to merge. --- lib/syskit/network_generation/merge_solver.rb | 206 ++++++++++++------ test/network_generation/test_merge_solver.rb | 146 ++++++++----- 2 files changed, 233 insertions(+), 119 deletions(-) diff --git a/lib/syskit/network_generation/merge_solver.rb b/lib/syskit/network_generation/merge_solver.rb index ee2d57527..197452ec0 100644 --- a/lib/syskit/network_generation/merge_solver.rb +++ b/lib/syskit/network_generation/merge_solver.rb @@ -286,11 +286,9 @@ def merge_task_contexts each_task_context_merge_candidate(task) do |merged_task| # Try to resolve the merge - can_merge, mappings = - resolve_merge(merged_task, task, merged_task => task) - - if can_merge - apply_merge_group(mappings) + result = resolve_merge(merged_task, task, merged_task => task) + if result.can_merge? + apply_merge_group(result.mappings) else invalid_merges << [merged_task, task] end @@ -412,44 +410,117 @@ def merge_compositions end end + # Representation of the result of {#resolve_merge} + MergeResolution = Struct.new( + :mappings, :merged_task, :task, :merged_failure_chain, :failure_chain + ) do + # Whether the merge resolution was successful + def can_merge? + !failure_chain + end + + # @!method mappings + # the set of known-good merges, only considering + # tasks' intrinsic properties + + # @!method failure_chain + # when the merge cannot be performed, a + # chain that links the original task pair given to resolve_merge + # up to two tasks that can't be merged, in the form + # `[t1, p1, t2, p2, t3, ... tN]``meaning that t2 is connected on t1's + # sink port p1, t3 is connected on t2's sink port p2 and tN + # is the task that cannot be merged + end + + Connection = Struct.new( + :source_task, :source_port, :policy, :sink_port, :sink_task + ) + + # Resolve merge between N tasks with the given tasks as seeds + # + # The method will cycle through the task's mismatching inputs (if + # there are any) and recursively resolve them, until it determines + # if the whole group can or cannot be merged + # + # @param [Syskit::TaskContext] merged_task task that is being considered + # to be merged into 'task' + # @param [Syskit::TaskContext] task the task into which merged_task will + # be merged. I.e. the operation is `task.merge(merged_task)`` + # @param [{Syskit::TaskContext=>Syskit::TaskContext}] mappings hash + # of "known good" merges that do not consider the dataflow (i.e. only + # considering intrinsic properties of the tasks themselves) + # + # @return [MergeResolution] def resolve_merge(merged_task, task, mappings) - mismatched_inputs = log_nest(2) { resolve_input_matching(merged_task, task) } + unless may_merge_task_contexts?(merged_task, task) + return MergeResolution.new(mappings, merged_task, task, [], []) + end + + mismatched_inputs, failed_connections = log_nest(2) do + resolve_input_matching(merged_task, task) + end + unless mismatched_inputs - # Incompatible inputs - return false, mappings + return MergeResolution.new( + mappings, merged_task, task, + [failed_connections[0]], [failed_connections[1]] + ) end - mismatched_inputs.each do |sink_port, merged_source_task, source_task| - info do - info " looking to pair the inputs of port #{sink_port} of" - info " #{merged_source_task}" - info " -- and --" - info " #{source_task}" - break - end + mappings = mappings.merge({ merged_task => task }) + mismatched_inputs.each do |connection, m_connection| + next unless (result = process_port_mismatch(connection, m_connection, mappings)) + return result unless result.can_merge? - if mappings[merged_source_task] == source_task - info " are already paired in the merge resolution: matching" - next - elsif !may_merge_task_contexts?(merged_source_task, source_task) - info " rejected: may not be merged" - return false, mappings - end + mappings = result.mappings + end - can_merge, mappings = log_nest(2) do - resolve_merge(merged_source_task, source_task, - mappings.merge(merged_source_task => source_task)) - end + MergeResolution.new(mappings, merged_task, task) + end - if can_merge - info " resolved" - else - info " rejected: cannot find mapping to merge both tasks" - return false, mappings - end + # @api private + # + # Process the mismatch between two connections as returned by + # {#resolve_input_matching} + # + # @param [Connection] connection the mismatching connection on the + # merge-target side + # @param [Connection] connection the mismatching connection on the + # to-be-merged side + # @return [MergeResolution] merge result + def process_port_mismatch(connection, m_connection, mappings) + sink_port = connection.sink_port + merged_source_task = m_connection.source_task + source_task = connection.source_task + + info do + info " looking to pair the inputs of port #{sink_port} of" + info " #{merged_source_task}" + info " -- and --" + info " #{source_task}" + break + end + + if mappings[merged_source_task] == source_task + info " are already paired in the merge resolution: matching" + return end - [true, mappings] + resolution = log_nest(2) do + resolve_merge(merged_source_task, source_task, mappings) + end + + if resolution.can_merge? + info " resolved" + resolution + else + info " rejected: cannot find mapping to merge both tasks" + MergeResolution.new( + resolution.mappings, m_connection.sink_task, connection.sink_task, + [connection] + resolution.failure_chain, + [m_connection] + resolution.merged_failure_chain + ) + end end def compatible_policies?(policy, other_policy) @@ -469,81 +540,86 @@ def compatible_policies?(policy, other_policy) # Otherwise, the set of mismatching inputs is returned, in which # each mismatch is a tuple (port_name,source_port,task_source,target_source). def resolve_input_matching(merged_task, task) - return [] if merged_task.equal?(task) + return [], nil if merged_task.equal?(task) m_inputs = Hash.new { |h, k| h[k] = {} } merged_task.each_concrete_input_connection do |m_source_task, m_source_port, sink_port, m_policy| - m_inputs[sink_port][[m_source_task, m_source_port]] = m_policy + m_inputs[sink_port][[m_source_task, m_source_port]] = + Connection.new(m_source_task, m_source_port, m_policy, + sink_port, merged_task) end - task.each_concrete_input_connection - .filter_map do |source_task, source_port, sink_port, policy| + mismatched_inputs = + task.each_concrete_input_connection + .filter_map do |source_task, source_port, sink_port, policy| # If merged_task has no connection on sink_port, the merge # is always valid next unless m_inputs.key?(sink_port) + connection = Connection.new( + source_task, source_port, policy, sink_port, task + ) + port_model = merged_task.model.find_input_port(sink_port) - resolved = + resolved, m_failed_connection = if port_model&.multiplexes? resolve_multiplexing_input( - sink_port, source_task, source_port, policy, - m_inputs[sink_port] + connection, m_inputs[sink_port] ) else - resolve_input( - sink_port, source_task, source_port, policy, - m_inputs[sink_port] - ) + resolve_input(connection, m_inputs[sink_port]) end - break unless resolved + unless resolved + return nil, [m_failed_connection, connection] + end resolved unless resolved.empty? end + + [mismatched_inputs, nil] end - def resolve_multiplexing_input( - sink_port, source_task, source_port, policy, m_inputs - ) - return [] unless (m_policy = m_inputs[[source_task, source_port]]) + def resolve_multiplexing_input(connection, m_inputs) + m_connection = m_inputs[[connection.source_task, connection.source_port]] + return [], nil unless m_connection # Already connected to the same task and port, we # just need to check whether the connections are # compatible - return [] if compatible_policies?(policy, m_policy) + return [], nil if compatible_policies?(connection.policy, m_connection.policy) debug do "rejected: incompatible policies on #{sink_port}" end - nil + [nil, m_connection] end - def resolve_input( - sink_port, source_task, source_port, policy, m_inputs - ) + def resolve_input(connection, m_inputs) # If we are not multiplexing, there can be only one source # for merged_task - (m_source_task, m_source_port), m_policy = m_inputs.first + m_connection = m_inputs.first.last - if m_source_port != source_port + if m_connection.source_port != connection.source_port debug do "rejected: sink #{sink_port} is connected to a port "\ - "named #{m_source_port}, expected #{source_port}" + "named #{m_connection.source_port}, expected "\ + "#{connection.source_port}" end - return + return nil, m_connection end - unless compatible_policies?(policy, m_policy) + unless compatible_policies?(connection.policy, m_connection.policy) debug do - "rejected: incompatible policies on #{sink_port}" + "rejected: incompatible policies on #{connection.sink_port}" end - return + return nil, m_connection end - if m_source_task == source_task - [] + if m_connection.source_task == connection.source_task + [[], nil] else - [sink_port, m_source_task, source_task] + [[connection, m_connection], nil] end end diff --git a/test/network_generation/test_merge_solver.rb b/test/network_generation/test_merge_solver.rb index c8ba125b5..e54fed0ad 100644 --- a/test/network_generation/test_merge_solver.rb +++ b/test/network_generation/test_merge_solver.rb @@ -6,6 +6,8 @@ describe Syskit::NetworkGeneration::MergeSolver do include Syskit::Fixtures::SimpleCompositionModel + connection_t = Syskit::NetworkGeneration::MergeSolver::Connection + attr_reader :solver before do @@ -155,11 +157,12 @@ .mock end - it "should return an empty array if given the same task" do + it "detects strictly identical inputs" do merged_task = mock_merged_task_with_concrete_input_connections( [src = Object.new, "src_port", "sink_port", {}] ) - assert_equal [], solver.resolve_input_matching(merged_task, merged_task) + assert_equal [[], nil], + solver.resolve_input_matching(merged_task, merged_task) end it "should not check for multiplexing ports for ports that do match" do merged_task = mock_merged_task_with_concrete_input_connections( @@ -168,44 +171,58 @@ task_model.should_receive(:find_input_port).never solver.resolve_input_matching(merged_task, merged_task) end - it "should return nil if the source port name is different" do + it "does not match inputs that have different source port names" do merged_task = mock_merged_task_with_concrete_input_connections( [src = Object.new, "src_port", "sink_port", {}] ) task = mock_task_with_concrete_input_connections( [src, "other_src_port", "sink_port", {}] ) - assert !solver.resolve_input_matching(merged_task, task) + m_expected_connection = connection_t.new( + src, "src_port", {}, "sink_port", merged_task + ) + expected_connection = connection_t.new( + src, "other_src_port", {}, "sink_port", task + ) + assert_equal [nil, [m_expected_connection, expected_connection]], + solver.resolve_input_matching(merged_task, task) end - it "should return nil if the policies are different" do - src_port = flexmock - merged_task = mock_merged_task_with_concrete_input_connections( - [src_port, "src_port", "sink_port", - merged_task_policy = flexmock(empty?: false)] + it "does not match if the policies are different" do + src = flexmock + m_task = mock_merged_task_with_concrete_input_connections( + [src, "src_port", "sink_port", m_policy = mock_policy] ) task = mock_task_with_concrete_input_connections( - [src_port, "src_port", "sink_port", - policy = flexmock(empty?: false)] + [src, "src_port", "sink_port", policy = mock_policy] ) flexmock(Syskit).should_receive(:update_connection_policy) - .with(merged_task_policy, policy).and_return(nil).once - refute solver.resolve_input_matching(merged_task, task) + .with(m_policy, policy).and_return(nil).once + m_connection = connection_t.new( + src, "src_port", m_policy, "sink_port", m_task + ) + connection = connection_t.new(src, "src_port", policy, "sink_port", task) + assert_equal [nil, [m_connection, connection]], + solver.resolve_input_matching(m_task, task) end - it "should call the task model with the input port name to get the port model if connections mismatch" do + it "should call the task model with the input port name "\ + "to get the port model if connections mismatch" do task_model.should_receive(:find_input_port) .with("sink_port").once.and_return(port_model) - merged_task = mock_merged_task_with_concrete_input_connections( - [flexmock, "src_port", "sink_port", - merged_task_policy = flexmock(empty?: false)] + m_task = mock_merged_task_with_concrete_input_connections( + [m_src = flexmock, "src_port", "sink_port", m_policy = mock_policy] ) task = mock_task_with_concrete_input_connections( - [flexmock, "src_port", "sink_port", - policy = flexmock(empty?: false)] + [src = flexmock, "src_port", "sink_port", policy = mock_policy] ) flexmock(Syskit).should_receive(:update_connection_policy) - .with(merged_task_policy, policy) - .and_return(nil).once - refute solver.resolve_input_matching(merged_task, task) + .with(m_policy, policy).and_return(nil).once + + m_connection = connection_t.new( + m_src, "src_port", m_policy, "sink_port", m_task + ) + connection = connection_t.new(src, "src_port", policy, "sink_port", task) + assert_equal [nil, [m_connection, connection]], + solver.resolve_input_matching(m_task, task) end it "returns the mismatching connection if the source port task is different" do merged_task = mock_merged_task_with_concrete_input_connections( @@ -214,35 +231,41 @@ task = mock_task_with_concrete_input_connections( [src = flexmock, "src_port", "sink_port", {}] ) - assert_equal [["sink_port", merged_task_src, src]], + m_connection = connection_t.new( + merged_task_src, "src_port", {}, "sink_port", merged_task + ) + connection = connection_t.new(src, "src_port", {}, "sink_port", task) + assert_equal [[[connection, m_connection]], nil], solver.resolve_input_matching(merged_task, task) end - it "returns nil if the source port tasks are different and the policies are not compatible" do - merged_task = mock_merged_task_with_concrete_input_connections( - [flexmock, "src_port", "sink_port", - merged_task_policy = flexmock(empty?: false)] + it "returns nil if the source port tasks are different "\ + "and the policies are not compatible" do + m_task = mock_merged_task_with_concrete_input_connections( + [m_src = flexmock, "src_port", "sink_port", m_policy = mock_policy] ) task = mock_task_with_concrete_input_connections( - [flexmock, "src_port", "sink_port", - policy = flexmock(empty?: false)] + [src = flexmock, "src_port", "sink_port", policy = mock_policy] + ) + m_connection = connection_t.new( + m_src, "src_port", m_policy, "sink_port", m_task ) + connection = connection_t.new(src, "src_port", policy, "sink_port", task) flexmock(Syskit).should_receive(:update_connection_policy) - .with(merged_task_policy, policy).and_return(nil).once - refute solver.resolve_input_matching(merged_task, task) + .with(m_policy, policy).and_return(nil).once + assert_equal [nil, [m_connection, connection]], + solver.resolve_input_matching(m_task, task) end it "returns an empty array if the source port is the same and "\ "the policies are compatible" do source_task = flexmock merged_task = mock_merged_task_with_concrete_input_connections( - [source_task, "src_port", "sink_port", - merged_task_policy = flexmock(empty?: false)] + [source_task, "src_port", "sink_port", merged_task_policy = mock_policy] ) task = mock_task_with_concrete_input_connections( - [source_task, "src_port", "sink_port", - policy = flexmock(empty?: false)] + [source_task, "src_port", "sink_port", policy = mock_policy] ) flexmock(solver).should_receive(compatible_policies?: true) - assert_equal([], solver.resolve_input_matching(merged_task, task)) + assert_equal [[], nil], solver.resolve_input_matching(merged_task, task) end describe "connections to multiplexing inputs" do @@ -255,18 +278,29 @@ .with(merged_task_policy, policy) .and_return(nil).by_default end - it "returns nil if connections from the same port are connected with "\ - "incompatible policies" do - src_port = flexmock + it "does not find a match if connections from the same port "\ + "are connected with incompatible policies" do + src_task = flexmock merged_task = mock_merged_task_with_concrete_input_connections( - [src_port, "src_port", "sink_port", merged_task_policy] + [src_task, "src_port", "sink_port", merged_task_policy] ) task = mock_task_with_concrete_input_connections( - [src_port, "src_port", "sink_port", policy] + [src_task, "src_port", "sink_port", policy] ) - refute solver.resolve_input_matching(merged_task, task) + inputs, m_failed_connections = + solver.resolve_input_matching(merged_task, task) + refute inputs + m_expected_connection = connection_t.new( + src_task, "src_port", merged_task_policy, "sink_port", merged_task + ) + expected_connection = connection_t.new( + src_task, "src_port", policy, "sink_port", task + ) + + assert_equal [m_expected_connection, expected_connection], + m_failed_connections end - it "returns an empty array if connections from the same port are "\ + it "finds a match if connections from the same port are "\ "connected with compatible policies" do src_port = flexmock merged_task = mock_merged_task_with_concrete_input_connections( @@ -276,29 +310,29 @@ [src_port, "src_port", "sink_port", policy] ) flexmock(solver).should_receive(compatible_policies?: true) - assert_equal [], solver.resolve_input_matching(merged_task, task) + assert_equal [[], nil], solver.resolve_input_matching(merged_task, task) end - it "returns an empty array for connections from different ports "\ + it "allows connections from different ports "\ "regardless of the connection policy" do - src_port = flexmock + src_task = flexmock merged_task = mock_merged_task_with_concrete_input_connections( - [src_port, "src_port", "sink_port", merged_task_policy] + [src_task, "src_port", "sink_port", merged_task_policy] ) task = mock_task_with_concrete_input_connections( - [src_port, "other_src_port", "sink_port", policy] + [src_task, "other_src_port", "sink_port", policy] ) - assert_equal [], solver.resolve_input_matching(merged_task, task) + assert_equal [[], nil], solver.resolve_input_matching(merged_task, task) end - it "returns an empty array for connections from different tasks "\ + it "allows connections from different tasks "\ "regardless of the connection policy" do - src_port = flexmock + src_task = flexmock merged_task = mock_merged_task_with_concrete_input_connections( - [src_port, "src_port", "sink_port", merged_task_policy] + [src_task, "src_port", "sink_port", merged_task_policy] ) task = mock_task_with_concrete_input_connections( - [src_port, "other_src_port", "sink_port", policy] + [src_task, "other_src_port", "sink_port", policy] ) - assert_equal [], solver.resolve_input_matching(merged_task, task) + assert_equal [[], nil], solver.resolve_input_matching(merged_task, task) end end @@ -373,4 +407,8 @@ def mock_merged_task_with_concrete_input_connections(*connections) assert_equal source_task, cmp.source_child end end + + def mock_policy + flexmock(empty?: false) + end end From 9c3e24893ddb5188b304330dde420a36c5e1fb34 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 31 May 2023 11:43:29 -0300 Subject: [PATCH 075/260] fix: display the whole merge chain that cause a device to be duplicated --- lib/syskit/exceptions.rb | 34 ++--- lib/syskit/network_generation/merge_solver.rb | 31 +++-- .../test_system_network_generator.rb | 19 +++ test/test_exceptions.rb | 119 +++++++++++++++++- 4 files changed, 159 insertions(+), 44 deletions(-) diff --git a/lib/syskit/exceptions.rb b/lib/syskit/exceptions.rb index 68410dded..70199b4a0 100644 --- a/lib/syskit/exceptions.rb +++ b/lib/syskit/exceptions.rb @@ -457,36 +457,16 @@ def can_merge? def initialize(device, task0, task1) @device = device @tasks = [task0, task1] - @can_merge = task0.can_merge?(task1) || task1.can_merge?(task0) - if can_merge? - # Mismatching inputs ... gather more info - @inputs = [] - @inputs[0] = task0.each_concrete_input_connection.to_a - @inputs[1] = task1.each_concrete_input_connection.to_a - end + + solver = NetworkGeneration::MergeSolver.new(task0.plan) + @merge_result = solver.resolve_merge(task0, task1, {}) end def pretty_print(pp) - pp.text "device #{device.name} is assigned to two tasks" - if can_merge? - pp.text " that have mismatching inputs" - tasks.each_with_index do |t, i| - pp.breakable - t.pretty_print(pp) - pp.breakable - pp.text "#{inputs[i].size} input(s):" - inputs[i].each do |source_task, source_port, sink_port| - pp.breakable - pp.text " #{source_task}.#{source_port} -> #{sink_port}" - end - end - else - pp.text " that cannot be merged" - tasks.each do |t| - pp.breakable - t.pretty_print(pp) - end - end + pp.text "device '#{device.name}' of type #{device.model} is assigned " + pp.text "to two tasks that cannot be merged" + pp.breakable + @merge_result.pretty_print_failure(pp) end end diff --git a/lib/syskit/network_generation/merge_solver.rb b/lib/syskit/network_generation/merge_solver.rb index 197452ec0..e17b15fe9 100644 --- a/lib/syskit/network_generation/merge_solver.rb +++ b/lib/syskit/network_generation/merge_solver.rb @@ -419,17 +419,26 @@ def can_merge? !failure_chain end - # @!method mappings - # the set of known-good merges, only considering - # tasks' intrinsic properties - - # @!method failure_chain - # when the merge cannot be performed, a - # chain that links the original task pair given to resolve_merge - # up to two tasks that can't be merged, in the form - # `[t1, p1, t2, p2, t3, ... tN]``meaning that t2 is connected on t1's - # sink port p1, t3 is connected on t2's sink port p2 and tN - # is the task that cannot be merged + def pretty_print_failure(pp) + pp.text "Chain 1 cannot be merged in chain 2:" + [[merged_task, merged_failure_chain], [task, failure_chain]] + .each_with_index do |(task, chain), i| + pp.breakable + pp.text "Chain #{i + 1}:" + pp.nest(2) do + pp.breakable + task.pretty_print(pp) + chain.each do |connection| + pp.breakable + pp.text "sink #{connection.sink_port}_port connected " + pp.text "via policy #{connection.policy} to source " + pp.text "#{connection.source_port}_port of" + pp.breakable + connection.source_task.pretty_print(pp) + end + end + end + end end Connection = Struct.new( diff --git a/test/network_generation/test_system_network_generator.rb b/test/network_generation/test_system_network_generator.rb index d73aee186..d1a04b458 100644 --- a/test/network_generation/test_system_network_generator.rb +++ b/test/network_generation/test_system_network_generator.rb @@ -163,6 +163,25 @@ def compute_system_network(*requirements) SystemNetworkGenerator.new(plan).validate_generated_network end end + + it "validates that devices are allocated at most once" do + device_m = Device.new_submodel(name: "D") + task_m = TaskContext.new_submodel(name: "T") + task_m.argument :arg + task_m.driver_for device_m, as: "test" + robot = Robot::RobotDefinition.new + robot.device device_m, as: "test" + + plan.add(task1 = task_m.new(arg: 1, test_dev: robot.test_dev)) + plan.add(task2 = task_m.new(arg: 2, test_dev: robot.test_dev)) + e = assert_raises(ConflictingDeviceAllocation) do + SystemNetworkGenerator.new(plan).validate_generated_network + end + + assert_equal Set[task1, task2], e.tasks.to_set + formatted = PP.pp(e, +"") + assert_equal "", formatted.gsub(//, "") + end end describe "#verify_no_multiplexing_connections" do diff --git a/test/test_exceptions.rb b/test/test_exceptions.rb index eeea96d7e..73b8f946c 100644 --- a/test/test_exceptions.rb +++ b/test/test_exceptions.rb @@ -2,12 +2,119 @@ require "syskit/test/self" -describe Syskit::InvalidAutoConnection do - describe "#pretty_print" do - it "should not raise" do - source = flexmock(each_output_port: [], each_input_port: []) - sink = flexmock(each_output_port: [], each_input_port: []) - PP.pp(Syskit::InvalidAutoConnection.new(source, sink), "".dup) +module Syskit + describe InvalidAutoConnection do + describe "#pretty_print" do + it "should not raise" do + source = flexmock(each_output_port: [], each_input_port: []) + sink = flexmock(each_output_port: [], each_input_port: []) + PP.pp(Syskit::InvalidAutoConnection.new(source, sink), "".dup) + end + end + end + + describe ConflictingDeviceAllocation do + it "displays the two driver tasks if they are the ones not mergeable" do + device_m = Device.new_submodel(name: "D") + driver_m = TaskContext.new_submodel(name: "T") + driver_m.driver_for device_m, as: "test" + robot = Robot::RobotDefinition.new + robot.device device_m, as: "test" + + plan.add(task1 = driver_m.new(arg: 1, test_dev: robot.test_dev)) + plan.add(task2 = driver_m.new(arg: 2, test_dev: robot.test_dev)) + e = assert_raises(ConflictingDeviceAllocation) do + NetworkGeneration::SystemNetworkGenerator + .new(plan).validate_generated_network + end + + assert_equal Set[task1, task2], e.tasks.to_set + formatted = PP.pp(e, +"") + + expected = <<~PP.chomp + device 'test' of type D is assigned to two tasks that cannot be merged + Chain 1 cannot be merged in chain 2: + Chain 1: + T + no owners + arguments: + arg: 2, + test_dev: MasterDeviceInstance(test[D]_dev), + conf: default(["default"]), + read_only: default(false) + Chain 2: + T + no owners + arguments: + arg: 1, + test_dev: MasterDeviceInstance(test[D]_dev), + conf: default(["default"]), + read_only: default(false) + PP + assert_equal expected, formatted.gsub(//, "").chomp + end + + it "displays merge chains to explain why devices are duplicated" do + device_m = Device.new_submodel(name: "D") + driver_m = TaskContext.new_submodel(name: "Driver") do + input_port "in", "/double" + end + driver_m.driver_for device_m, as: "test" + task_m = TaskContext.new_submodel(name: "Task") do + argument :arg + output_port "out", "/double" + end + + robot = Robot::RobotDefinition.new + robot.device device_m, as: "test" + + plan.add(driver1 = driver_m.new(test_dev: robot.test_dev)) + plan.add(driver2 = driver_m.new(test_dev: robot.test_dev)) + plan.add(task1 = task_m.new(arg: 1)) + plan.add(task2 = task_m.new(arg: 2)) + task1.out_port.connect_to driver1.in_port + task2.out_port.connect_to driver2.in_port + e = assert_raises(ConflictingDeviceAllocation) do + NetworkGeneration::SystemNetworkGenerator + .new(plan).validate_generated_network + end + + assert_equal Set[driver1, driver2], e.tasks.to_set + formatted = PP.pp(e, +"") + + expected = <<~PP.chomp + device 'test' of type D is assigned to two tasks that cannot be merged + Chain 1 cannot be merged in chain 2: + Chain 1: + Driver + no owners + arguments: + test_dev: MasterDeviceInstance(test[D]_dev), + conf: default(["default"]), + read_only: default(false) + sink in_port connected via policy {} to source out_port of + Task + no owners + arguments: + arg: 1, + conf: default(["default"]), + read_only: default(false) + Chain 2: + Driver + no owners + arguments: + test_dev: MasterDeviceInstance(test[D]_dev), + conf: default(["default"]), + read_only: default(false) + sink in_port connected via policy {} to source out_port of + Task + no owners + arguments: + arg: 2, + conf: default(["default"]), + read_only: default(false) + PP + assert_equal expected, formatted.gsub(//, "").chomp end end end From eac9b1ef7236e3de1e00b2ee6f986ea6ee790f59 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 31 May 2023 18:46:37 -0300 Subject: [PATCH 076/260] feat: improve display of profile and profile definitions --- lib/syskit/actions/profile.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/syskit/actions/profile.rb b/lib/syskit/actions/profile.rb index ef82199a2..e26961e34 100644 --- a/lib/syskit/actions/profile.rb +++ b/lib/syskit/actions/profile.rb @@ -68,6 +68,10 @@ def resolve result.doc(doc) result end + + def to_s + "#{profile}.#{name}_def" + end end # Instance-level API for tags @@ -265,7 +269,7 @@ def resolved_dependency_injection end def to_s - "profile:#{name}" + name end # Promote requirements taken from another profile to this profile From 3734bf647c6abda61e62dc992b7db88d997662df Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 31 May 2023 18:47:11 -0300 Subject: [PATCH 077/260] feat: show involved definitions when describing conflicting device allocations --- lib/syskit/actions/profile.rb | 8 +-- lib/syskit/exceptions.rb | 26 +++++++++- .../system_network_generator.rb | 21 ++++++-- test/test_exceptions.rb | 49 +++++++++++++++++++ 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/lib/syskit/actions/profile.rb b/lib/syskit/actions/profile.rb index e26961e34..a6198dc1f 100644 --- a/lib/syskit/actions/profile.rb +++ b/lib/syskit/actions/profile.rb @@ -53,6 +53,10 @@ def to_action_model(profile = self.profile, doc = self.doc) action_model.advanced = advanced? action_model end + + def to_s + "#{profile}.#{name}_def" + end end class Definition < ProfileInstanceRequirements @@ -68,10 +72,6 @@ def resolve result.doc(doc) result end - - def to_s - "#{profile}.#{name}_def" - end end # Instance-level API for tags diff --git a/lib/syskit/exceptions.rb b/lib/syskit/exceptions.rb index 70199b4a0..77f13a6b6 100644 --- a/lib/syskit/exceptions.rb +++ b/lib/syskit/exceptions.rb @@ -454,12 +454,24 @@ def can_merge? !!@can_merge end - def initialize(device, task0, task1) + def initialize(device, task0, task1, toplevel_tasks_to_requirements = {}) @device = device @tasks = [task0, task1] solver = NetworkGeneration::MergeSolver.new(task0.plan) @merge_result = solver.resolve_merge(task0, task1, {}) + @involved_definitions = @tasks.map do |t| + find_all_related_syskit_actions(t, toplevel_tasks_to_requirements) + end + end + + def find_all_related_syskit_actions(task, toplevel_tasks_to_requirements) + result = [] + while task + result.concat(toplevel_tasks_to_requirements[task] || []) + task = task.each_parent_task.first + end + result end def pretty_print(pp) @@ -467,6 +479,18 @@ def pretty_print(pp) pp.text "to two tasks that cannot be merged" pp.breakable @merge_result.pretty_print_failure(pp) + @involved_definitions.each_with_index do |defs, i| + next if defs.empty? + + pp.breakable + pp.text "Chain #{i + 1} is needed by the following definitions:" + pp.nest(2) do + defs.each do |d| + pp.breakable + pp.text d.to_s + end + end + end end end diff --git a/lib/syskit/network_generation/system_network_generator.rb b/lib/syskit/network_generation/system_network_generator.rb index 85273aaef..aa7afd691 100644 --- a/lib/syskit/network_generation/system_network_generator.rb +++ b/lib/syskit/network_generation/system_network_generator.rb @@ -187,9 +187,10 @@ def self.remove_abstract_composition_optional_children(plan) def compute_system_network(instance_requirements, garbage_collect: true, validate_abstract_network: true, validate_generated_network: true) - toplevel_tasks = log_timepoint_group "instanciate" do + @toplevel_tasks = log_timepoint_group "instanciate" do instanciate(instance_requirements) end + @toplevel_instance_requirements = instance_requirements merge_solver.merge_identical_tasks log_timepoint "merge" @@ -237,12 +238,20 @@ def compute_system_network(instance_requirements, garbage_collect: true, self.validate_abstract_network log_timepoint "validate_abstract_network" end + if validate_generated_network self.validate_generated_network log_timepoint "validate_generated_network" end - toplevel_tasks + @toplevel_tasks + end + + def toplevel_tasks_to_requirements + (@toplevel_tasks || []) + .map { |t| merge_solver.replacement_for(t) } + .zip(@toplevel_instance_requirements || []) + .each_with_object({}) { |(t, ir), h| (h[t] ||= []) << ir } end # Verifies that the task allocation is complete @@ -298,7 +307,7 @@ def self.verify_no_multiplexing_connections(plan) # attached to any device # @raise [SpecError] if some devices are assigned to more than one # task - def self.verify_device_allocation(plan) + def self.verify_device_allocation(plan, toplevel_tasks_to_requirements = {}) components = plan.find_local_tasks(Syskit::Device).to_a # Check that all devices are properly assigned @@ -317,7 +326,9 @@ def self.verify_device_allocation(plan) task.each_master_device do |dev| device_name = dev.full_name if (old_task = devices[device_name]) - raise ConflictingDeviceAllocation.new(dev, task, old_task) + raise ConflictingDeviceAllocation.new( + dev, task, old_task, toplevel_tasks_to_requirements + ) else devices[device_name] = task end @@ -337,7 +348,7 @@ def validate_abstract_network # Validates the network generated by {#compute_system_network} def validate_generated_network self.class.verify_task_allocation(plan) - self.class.verify_device_allocation(plan) + self.class.verify_device_allocation(plan, toplevel_tasks_to_requirements) super if defined? super end end diff --git a/test/test_exceptions.rb b/test/test_exceptions.rb index 73b8f946c..af3b71b08 100644 --- a/test/test_exceptions.rb +++ b/test/test_exceptions.rb @@ -54,6 +54,55 @@ module Syskit assert_equal expected, formatted.gsub(//, "").chomp end + it "displays definitions that depend on the conflicting tasks" do + device_m = Device.new_submodel(name: "D") + driver_m = TaskContext.new_submodel(name: "T") + driver_m.argument :arg + driver_m.driver_for device_m, as: "test" + robot = Robot::RobotDefinition.new + robot.device device_m, as: "test" + + cmp_m = Composition.new_submodel + cmp_m.add device_m, as: "test" + profile = Actions::Profile.new("Test") + profile.define("test1", cmp_m) + .use("test" => robot.test_dev.with_arguments(arg: 1)) + profile.define("test2", cmp_m) + .use("test" => robot.test_dev.with_arguments(arg: 2)) + + self.syskit_run_planner_validate_network = true + e = assert_raises(ConflictingDeviceAllocation) do + run_planners([profile.test1_def, profile.test2_def]) + end + + formatted = PP.pp(e, +"") + expected = <<~PP.chomp + device 'test' of type D is assigned to two tasks that cannot be merged + Chain 1 cannot be merged in chain 2: + Chain 1: + T + no owners + arguments: + test_dev: MasterDeviceInstance(test[D]_dev), + arg: 2, + conf: ["default"], + read_only: false + Chain 2: + T + no owners + arguments: + test_dev: MasterDeviceInstance(test[D]_dev), + arg: 1, + conf: ["default"], + read_only: false + Chain 1 is needed by the following definitions: + Test.test2_def + Chain 2 is needed by the following definitions: + Test.test1_def + PP + assert_equal expected, formatted.gsub(//, "").chomp + end + it "displays merge chains to explain why devices are duplicated" do device_m = Device.new_submodel(name: "D") driver_m = TaskContext.new_submodel(name: "Driver") do From ca02dc5a348833b6da217a2dfc3fc2df3b78da3b Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Jun 2023 10:00:18 -0300 Subject: [PATCH 078/260] fix: do not let PlanningFailedError and MissionFailedError leak out of syskit_deploy This tremendously reduces the amount of noise in failed profile tests. --- lib/syskit/test/network_manipulation.rb | 2 +- test/test/test_network_manipulation.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/syskit/test/network_manipulation.rb b/lib/syskit/test/network_manipulation.rb index e57b27a98..d4540000a 100644 --- a/lib/syskit/test/network_manipulation.rb +++ b/lib/syskit/test/network_manipulation.rb @@ -240,7 +240,7 @@ def syskit_deploy_normalized_placeholder_tasks( end.to do requirement_tasks.each do |t| have_error_matching(Roby::PlanningFailedError - .match.with_origin(t)) + .match.with_origin(t.planned_task)) end end raise diff --git a/test/test/test_network_manipulation.rb b/test/test/test_network_manipulation.rb index 02e2aacdc..acf51b33b 100644 --- a/test/test/test_network_manipulation.rb +++ b/test/test/test_network_manipulation.rb @@ -232,6 +232,14 @@ module Test task = syskit_deploy(@task_m) assert_equal "test_level", task.orocos_name end + + it "on error, it filters out the planning failed and mission failed "\ + "error caused by itself" do + task_m = Syskit::TaskContext.new_submodel + assert_raises(MissingDeployments) do + syskit_deploy(task_m) + end + end end describe "#syskit_generate_network" do From e7ee456d354dc8f9d66ad8aff9afab333dc424b4 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 1 Jun 2023 21:57:51 -0300 Subject: [PATCH 079/260] feat: display the type of assertion in ProfileAssertionFailed messages --- lib/syskit/test/profile_assertions.rb | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/syskit/test/profile_assertions.rb b/lib/syskit/test/profile_assertions.rb index 73ae4c880..7635739ba 100644 --- a/lib/syskit/test/profile_assertions.rb +++ b/lib/syskit/test/profile_assertions.rb @@ -15,13 +15,14 @@ module ProfileAssertions class ProfileAssertionFailed < Roby::ExceptionBase attr_reader :actions - def initialize(act, original_error) - @actions = Array(act) + def initialize(assertion_name, actions, original_error) + @assertion_name = assertion_name + @actions = Array(actions) super([original_error]) end def pretty_print(pp) - pp.text "Failure while running an assertion on" + pp.text "Failure while running a #{@assertion_name} assertion on" pp.nest(2) do actions.each do |act| pp.breakable @@ -270,7 +271,8 @@ def syskit_assert_action_is_self_contained( plan.unmark_mission_task(task) expect_execution.garbage_collect(true).to_run rescue Minitest::Assertion, StandardError => e - raise ProfileAssertionFailed.new(action, e), e.message, e.backtrace + raise ProfileAssertionFailed.new("self contained", action, e), + e.message, e.backtrace end # Spec-style call for {#assert_is_self_contained} @@ -352,7 +354,8 @@ def assert_can_instanciate_together(*actions) actions, compute_policies: false, compute_deployments: false ) rescue Minitest::Assertion, StandardError => e - raise ProfileAssertionFailed.new(actions, e), e.message, e.backtrace + raise ProfileAssertionFailed.new("instanciate together", actions, e), + e.message, e.backtrace end # Spec-style call for {#assert_can_instanciate_together} @@ -458,7 +461,8 @@ def assert_can_deploy_together(*actions) actions, compute_policies: true, compute_deployments: true ) rescue Minitest::Assertion, StandardError => e - raise ProfileAssertionFailed.new(actions, e), e.message, e.backtrace + raise ProfileAssertionFailed.new("deploy together", actions, e), + e.message, e.backtrace end def syskit_run_deploy_in_bulk( @@ -533,15 +537,17 @@ def assert_can_configure_together(*actions) task_contexts = tasks.find_all { |t| t.kind_of?(Syskit::TaskContext) } .each do |task_context| unless task_context.plan - raise ProfileAssertionFailed.new(actions, nil), - "#{task_context} got garbage-collected before "\ - "it got configured" + raise ProfileAssertionFailed.new( + "configure together", actions, nil + ), "#{task_context} got garbage-collected before "\ + "it got configured" end end syskit_configure(task_contexts) roots rescue Minitest::Assertion, StandardError => e - raise ProfileAssertionFailed.new(actions, e), e.message, e.backtrace + raise ProfileAssertionFailed.new("configure together", actions, e), + e.message, e.backtrace end # Spec-style call for {#assert_can_configure_together} From 7e9316e76f749f260b53caf05664f347279f7562 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Fri, 16 Jun 2023 19:29:09 -0300 Subject: [PATCH 080/260] fix: correct last remaining failing tests --- test/live/test_fatal_error.rb | 4 +--- test/models/test_composition_child.rb | 14 +++++++----- .../test_system_network_generator.rb | 22 ++++++++++++++++++- test/test/test_profile_assertions.rb | 22 +++++++++++-------- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/test/live/test_fatal_error.rb b/test/live/test_fatal_error.rb index 42f310e0c..b1d2ec883 100644 --- a/test/live/test_fatal_error.rb +++ b/test/live/test_fatal_error.rb @@ -44,11 +44,9 @@ class SyskitFatalErrorTests < Syskit::Test::ComponentTest Syskit.conf.auto_restart_deployments_with_quarantines = false trigger_fatal_error(@task) - e = assert_raises(Roby::Test::ExecutionExpectations::UnexpectedErrors) do + assert_raises(TaskContextInFatal) do syskit_deploy(@task_m) end - assert_kind_of Roby::PlanningFailedError, - e.each_execution_exception.first.exception end it "auto-restarts deployments with a task in FATAL_ERROR "\ diff --git a/test/models/test_composition_child.rb b/test/models/test_composition_child.rb index 7302866cb..168dbc077 100644 --- a/test/models/test_composition_child.rb +++ b/test/models/test_composition_child.rb @@ -161,8 +161,8 @@ @cmp_m.test_child.bind(cmp.test_child.other_srv) end assert_equal "cannot bind Syskit::Models::Placeholder to "\ - "#", - e.message.gsub(/0x[0-9a-f]+/, "0x") + "#", + e.message end end @@ -189,9 +189,10 @@ @cmp_m.test_child.bind(task) end assert_equal "cannot bind cmp_m.test_child["\ - "Syskit::Models::Placeholder] to task_m:: "\ + "Syskit::Models::Placeholder] to task_m"\ + "(conf: default([\"default\"]), read_only: default(false)): "\ "it is not the child of any cmp_m composition", - e.message.gsub(/:0x.*:/, "::") + e.message.gsub(//, "") end it "does not move up to parents when the role does not match" do plan.add(task = @task_m.new) @@ -201,9 +202,10 @@ @cmp_m.test_child.bind(task) end assert_equal "cannot bind cmp_m.test_child[Syskit::Models::"\ - "Placeholder] to task_m:: it is the child of one "\ + "Placeholder] to task_m(conf: default([\"default\"]), "\ + "read_only: default(false)): it is the child of one "\ "or more cmp_m compositions, but not with the role 'test'", - e.message.gsub(/:0x.*:/, "::") + e.message.gsub(//, "") end end end diff --git a/test/network_generation/test_system_network_generator.rb b/test/network_generation/test_system_network_generator.rb index d1a04b458..bcd21e753 100644 --- a/test/network_generation/test_system_network_generator.rb +++ b/test/network_generation/test_system_network_generator.rb @@ -180,7 +180,27 @@ def compute_system_network(*requirements) assert_equal Set[task1, task2], e.tasks.to_set formatted = PP.pp(e, +"") - assert_equal "", formatted.gsub(//, "") + expected = <<~MSG + device 'test' of type D is assigned to two tasks that cannot be merged + Chain 1 cannot be merged in chain 2: + Chain 1: + T + no owners + arguments: + arg: 2, + test_dev: MasterDeviceInstance(test[D]_dev), + conf: default(["default"]), + read_only: default(false) + Chain 2: + T + no owners + arguments: + arg: 1, + test_dev: MasterDeviceInstance(test[D]_dev), + conf: default(["default"]), + read_only: default(false) + MSG + assert_equal expected, formatted.gsub(//, "") end end diff --git a/test/test/test_profile_assertions.rb b/test/test/test_profile_assertions.rb index 56d7b1e66..57aa19161 100644 --- a/test/test/test_profile_assertions.rb +++ b/test/test/test_profile_assertions.rb @@ -345,7 +345,8 @@ module Test end assert_match( /cannot\ find\ a\ concrete\ implementation.* - TestProfile.test_tag/mx, e.message + TestProfile.test_tag/mx, + PP.pp(e.each_original_exception.first, +"") ) end @@ -356,7 +357,8 @@ module Test end assert_match( /cannot\ find\ a\ concrete\ implementation.* - Models::Placeholder/mx, e.message + Models::Placeholder/mx, + PP.pp(e.each_original_exception.first, +"") ) end @@ -371,8 +373,8 @@ module Test assert_can_instanciate(@test_profile) end assert_match( - /cannot find a concrete implementation.*profile:Other.test_tag/m, - e.message + /cannot find a concrete implementation.*Other.test_tag/m, + PP.pp(e.each_original_exception.first, +"") ) end @@ -465,7 +467,7 @@ module Test assert_match( /cannot deploy the following tasks.*Task.*child test of Cmp/m, - e.message + PP.pp(e.each_original_exception.first, +"") ) end @@ -479,7 +481,8 @@ module Test end assert_match( /cannot\ find\ a\ concrete\ implementation.* - TestProfile.test_tag/mx, e.message + TestProfile.test_tag/mx, + PP.pp(e.each_original_exception.first, +"") ) end @@ -490,7 +493,8 @@ module Test end assert_match( /cannot\ find\ a\ concrete\ implementation.* - Models::Placeholder/mx, e.message + Models::Placeholder/mx, + PP.pp(e.each_original_exception.first, +"") ) end @@ -505,8 +509,8 @@ module Test assert_can_deploy(@test_profile) end assert_match( - /cannot find a concrete implementation.*profile:Other.test_tag/m, - e.message + /cannot find a concrete implementation.*Other.test_tag/m, + PP.pp(e.each_original_exception.first, +"") ) end From cbb2b8eed0d9d47ef0dffa8ceffcb99d488a0a5e Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 26 Jun 2023 16:54:10 -0300 Subject: [PATCH 081/260] fix: only consider concrete connections when setting up the logger This code has essentially only ever been used to setup default loggers, which are always directly connected to the component they are logging. However, when using the logger manually, we do want to be able to connect it through composition(s). Fix this use-case --- lib/syskit/network_generation/logger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syskit/network_generation/logger.rb b/lib/syskit/network_generation/logger.rb index 855ba0379..ff0ef2092 100644 --- a/lib/syskit/network_generation/logger.rb +++ b/lib/syskit/network_generation/logger.rb @@ -65,7 +65,7 @@ def create_logging_port(sink_port_name, logged_task, logged_port) def configure super - each_input_connection do |source_task, source_port_name, sink_port_name, policy| + each_concrete_input_connection do |source_task, source_port_name, sink_port_name, policy| source_port = source_task.find_output_port(source_port_name) create_logging_port(sink_port_name, source_task, source_port) end From 29a0a5aee17c3945be2e57cb83b3a3db03a1b737 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 18 Jul 2023 09:27:26 -0300 Subject: [PATCH 082/260] feat: implement syskit log_runtime_archive A tool meant to incrementally archive a syskit dataset as it is being generated in a tarball. It compresses each file separately with zstd for simplicity reasons. --- bin/syskit | 2 +- lib/syskit/cli/log_runtime_archive.rb | 195 ++++++++++++++++++++++ lib/syskit/scripts/log_runtime_archive.rb | 30 ++++ test/cli/test_log_runtime_archive.rb | 154 +++++++++++++++++ 4 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 lib/syskit/cli/log_runtime_archive.rb create mode 100644 lib/syskit/scripts/log_runtime_archive.rb create mode 100644 test/cli/test_log_runtime_archive.rb diff --git a/bin/syskit b/bin/syskit index 349e87338..e4d170e3b 100755 --- a/bin/syskit +++ b/bin/syskit @@ -52,7 +52,7 @@ end ORIGINAL_ARGV = ARGV.dup mode = ARGV.shift -SYSKIT_MODES = %w[ide process_server].freeze +SYSKIT_MODES = %w[ide process_server log_runtime_archive].freeze ROBY_MODES = %w[run shell test gen quit restart].freeze if [nil, "--help", "-h", "help"].include?(mode) thor_modes = Syskit::CLI::Main diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb new file mode 100644 index 000000000..4733135b1 --- /dev/null +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -0,0 +1,195 @@ +# frozen_string_literal: true + +require "archive/tar/minitar" + +module Syskit + module CLI + module LogRuntimeArchive + # Find all dataset-looking folders within a root log folder + def self.find_all_dataset_folders(root_dir) + candidates = root_dir.enum_for(:each_entry).map do |child| + next unless /^\d{8}\-\d{4}(\.\d+)?$/.match?(child.basename.to_s) + + child = (root_dir / child) + next unless child.directory? + + child if (child / "info.yml").file? + end + + candidates.compact.sort_by { _1.basename.to_s } + end + + # Safely add an entry into an archive, compressing it with zstd + def self.add_to_archive(archive_io, child_path) + puts "adding #{child_path}" + stat = child_path.stat + + start_pos = archive_io.tell + write_initial_header(archive_io, child_path, stat) + data_pos = archive_io.tell + exit_status = write_compressed_data(child_path, archive_io) + + if exit_status.success? + add_to_archive_commit(archive_io, child_path, start_pos, data_pos, stat) + child_path.unlink + else + add_to_archive_rollback(archive_io, start_pos) + end + + rescue Exception => e # rubocop:disable Lint/RescueException + add_to_archive_rollback(archive_io, start_pos) if start_pos + raise + end + + # Finalize appending a file in the archive + def self.add_to_archive_commit( + archive_io, child_path, start_pos, data_pos, stat + ) + data_size = archive_io.tell - data_pos + write_padding(data_size, archive_io) + + # Update header + archive_io.seek(start_pos, IO::SEEK_SET) + write_final_header(archive_io, child_path, stat, data_size) + archive_io.seek(0, IO::SEEK_END) + end + + # Revert the addition of a file in the archive, after an error + def self.add_to_archive_rollback(archive_io, start_pos) + archive_io.truncate(start_pos) + archive_io.seek(start_pos, IO::SEEK_SET) + end + + # Write a tar block header without the data size + def self.write_initial_header(archive_io, child_path, stat) + write_header( + archive_io, + "#{child_path.basename}.zst", + { mode: 0o644, uid: stat.uid, gid: stat.gid, + mtime: stat.mtime, size: 0 } + ) + end + + # Write the final tar block header at the given position + def self.write_final_header(archive_io, child_path, stat, size) + write_header( + archive_io, + "#{child_path.basename}.zst", + { mode: 0o644, uid: stat.uid, gid: stat.gid, + mtime: stat.mtime, size: size } + ) + end + + # Compress data and append it to the archive + def self.write_compressed_data(child_path, archive_io) + _, exit_status = child_path.open("r") do |io| + zstd_transfer_r, zstd_transfer_w = IO.pipe + pid = Process.spawn("zstd", "--stdout", in: io, out: zstd_transfer_w) + zstd_transfer_w.close + IO.copy_stream(zstd_transfer_r, archive_io) + Process.waitpid2(pid) + end + exit_status + end + + # Write necessary padding (tar requires multiples of 512 bytes) + def self.write_padding(size, io) + # Move to end, compute actual size, pad to 512 bytes blocks + remainder = (size + 511) / 512 * 512 - size + io.write("\0" * remainder) + end + + def self.archive_dataset(archive_io, path, full:) + puts "Archiving dataset #{path} in #{full ? "full" : "partial"} mode" + candidates = path.enum_for(:each_entry).map { path / _1 } + candidates = archive_partial_filter_candidates(candidates) unless full + + candidates.each do |child_path| + add_to_archive(archive_io, child_path) + end + end + + def self.archive_partial_filter_candidates(candidates) + per_file_and_idx = candidates.each_with_object({}) do |path, h| + name = path.basename.to_s + if (m = /\.(\d+)\.log$/.match(name)) + per_file = (h[m.pre_match] ||= {}) + per_file[Integer(m[1])] = path + end + end + + per_file_and_idx.each_value.flat_map do |logs| + logs.delete(logs.keys.max) + logs.values + end + end + + def self.process_root_folder(root_dir, target_dir) + candidates = find_all_dataset_folders(root_dir) + running = candidates.last + candidates.each do |child| + archive_path = target_dir / "#{child.basename}.tar" + mode = + if archive_path.exist? + "r+" + else + "w" + end + + archive_path.open(mode) do |archive_io| + archive_io.seek(0, IO::SEEK_END) + archive_dataset(archive_io, child, full: child != running) + end + end + end + + extend Archive::Tar::Minitar::ByteSize + + # Copy a tar header (copied from minitar) + def self.write_header(io, long_name, header) + short_name, prefix, needs_long_name = split_name(long_name) + + if needs_long_name + long_name_header = { + prefix: "", + name: Archive::Tar::Minitar::PosixHeader::GNU_EXT_LONG_LINK, + typeflag: "L", + size: long_name.length, + mode: 0 + } + io.write(Archive::Tar::Minitar::PosixHeader.new(long_name_header)) + io.write(long_name) + io.write("\0" * (512 - (long_name.length % 512))) + end + + new_header = header.merge({ name: short_name, prefix: prefix }) + io.write(Archive::Tar::Minitar::PosixHeader.new(new_header)) + end + + # Process a file name to determine whether it should use the GNU + # long file extension. Copied from minitar + def self.split_name(name) # rubocop:disable Metrics/AbcSize + if bytesize(name) <= 100 + prefix = "" + else + parts = name.split(%r{\/}) + newname = parts.pop + + nxt = "" + + loop do + nxt = parts.pop || "" + break if bytesize(newname) + 1 + bytesize(nxt) >= 100 + + newname = "#{nxt}/#{newname}" + end + + prefix = (parts + [nxt]).join('/') + name = newname + end + + [name, prefix, (bytesize(name) > 100 || bytesize(prefix) > 155)] + end + end + end +end diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb new file mode 100644 index 000000000..9f9b4be42 --- /dev/null +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -0,0 +1,30 @@ +#! /usr/bin/env ruby +# frozen_string_literal: true + +require "pathname" +require "syskit/cli/log_runtime_archive" + +unless ARGV.size == 2 + STDERR.puts "usage: log_runtime_archive ROOT_DIR TARGET_DIR" + exit 1 +end + +root_dir = Pathname.new(ARGV[0]) +target_dir = Pathname.new(ARGV[1]) + +if !root_dir.directory? + warn "#{root_dir} is not a directory" + exit 1 +elsif !target_dir.directory? + warn "#{target_dir} is not a directory" + exit 1 +end + +POLLING_PERIOD = 600 + +loop do + Syskit::CLI::LogRuntimeArchive.process_root_folder(root_dir, target_dir) + + puts "Archived pending logs, sleeping #{POLLING_PERIOD}s" + sleep POLLING_PERIOD +end diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb new file mode 100644 index 000000000..54d182041 --- /dev/null +++ b/test/cli/test_log_runtime_archive.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +require "syskit/test/self" +require "syskit/cli/log_runtime_archive" + +module Syskit + module CLI + describe LogRuntimeArchive do + describe ".find_all_dataset_folders" do + before do + @root = make_tmppath + end + + def make_valid_folder(name) + path = (@root / name) + path.mkpath + FileUtils.touch(path / "info.yml") + path + end + + it "returns the directories that look like a dataset path" do + path = make_valid_folder("20229523-1104.1") + assert_equal( + [path], + LogRuntimeArchive.find_all_dataset_folders(@root) + ) + end + + it "sorts the entries lexicographically" do + path1 = make_valid_folder("20229523-1104.1") + path2 = make_valid_folder("20229423-1104.1") + assert_equal( + [path2, path1], + LogRuntimeArchive.find_all_dataset_folders(@root) + ) + end + + it "does not return paths that match the pattern but "\ + "do not have a info.yml file inside" do + path = (@root / "20229423-1104") + path.mkpath + assert_equal([], LogRuntimeArchive.find_all_dataset_folders(@root)) + end + + it "does not return paths that do not match the pattern" do + path = (@root / "20240223-11043") + path.mkpath + FileUtils.touch(path / "info.yml") + assert_equal([], LogRuntimeArchive.find_all_dataset_folders(@root)) + end + + it "ignores paths that match the pattern but are not folders" do + path = (@root / "20240223-1104") + FileUtils.touch(path.to_s) + assert_equal([], LogRuntimeArchive.find_all_dataset_folders(@root)) + end + end + + describe ".add_to_archive" do + before do + @root = make_tmppath + @archive_path = (make_tmppath / "archive.tar") + @in_files = [] + end + + def make_in_file(name, content) + path = (@root / name) + path.write(content) + @in_files << path + path + end + + def read_archive + tar = Archive::Tar::Minitar::Input.open(@archive_path.open("r")) + tar.each_entry.map { |e| [e, e.read] } + end + + def decompress_data(data) + IO.popen(["zstd", "-d", "--stdout"], "r+") do |io| + io.write data + io.close_write + io.read + end + end + + def assert_entry_matches(entry, data, name:, content:) + assert entry.file? + assert_equal name, entry.full_name + assert_equal content, decompress_data(data) + end + + it "adds a compressed version of the input I/O to the archive "\ + "and deletes the input file" do + in_path = make_in_file "something.txt", "something" + + @archive_path.open("w") do |archive_io| + @in_files.each do |in_path| + LogRuntimeArchive.add_to_archive(archive_io, in_path) + end + end + + entries = read_archive + assert_equal 1, entries.size + assert_entry_matches( + *entries.first, name: "something.txt.zst", content: "something" + ) + refute in_path.exist? + end + + it "creates a multi-file archive" do + bla = make_in_file "bla.txt", "bla" + blo = make_in_file "blo.txt", "blo" + @archive_path.open("w") do |archive_io| + @in_files.each do |in_path| + LogRuntimeArchive.add_to_archive(archive_io, in_path) + end + end + + entries = read_archive + assert_equal 2, entries.size + assert_entry_matches(*entries[0], name: "bla.txt.zst", content: "bla") + assert_entry_matches(*entries[1], name: "blo.txt.zst", content: "blo") + refute bla.exist? + refute blo.exist? + end + + it "restores the file as it was and keeps the input file if zstd fails "\ + "but continues with other files" do + bla = make_in_file "bla.txt", "bla" + blo = make_in_file "blo.txt", "blo" + bli = make_in_file "bli.txt", "bli" + + @archive_path.open("w") do |archive_io| + LogRuntimeArchive.add_to_archive(archive_io, bla) + FlexMock.use(Process) do |mock| + mock.should_receive(:waitpid2).once + .and_return([10, flexmock(success?: false)]) + LogRuntimeArchive.add_to_archive(archive_io, blo) + end + LogRuntimeArchive.add_to_archive(archive_io, bli) + end + + entries = read_archive + assert_equal 2, entries.size + assert_entry_matches(*entries[0], name: "bla.txt.zst", content: "bla") + assert_entry_matches(*entries[1], name: "bli.txt.zst", content: "bli") + refute bla.exist? + assert blo.exist? + refute bli.exist? + end + end + end + end +end From 0cb06e80212f909e976ff383ad60e806b2e23a26 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 18 Jul 2023 15:06:20 -0300 Subject: [PATCH 083/260] fix: documentation and rubocop for log_runtime_archive --- lib/syskit/cli/log_runtime_archive.rb | 54 +++++++++++++++++++---- lib/syskit/scripts/log_runtime_archive.rb | 1 - test/cli/test_log_runtime_archive.rb | 4 +- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 4733135b1..e6ca1e509 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -4,6 +4,12 @@ module Syskit module CLI + # Implementation of the `syskit log-runtime-archive` tool + # + # The tool archives Syskit log directories into tar archives in realtime, + # compressing files using zstd + # + # It depends on the syskit instance using log rotation module LogRuntimeArchive # Find all dataset-looking folders within a root log folder def self.find_all_dataset_folders(root_dir) @@ -20,6 +26,8 @@ def self.find_all_dataset_folders(root_dir) end # Safely add an entry into an archive, compressing it with zstd + # + # @return [Boolean] true if the file was added successfully, false otherwise def self.add_to_archive(archive_io, child_path) puts "adding #{child_path}" stat = child_path.stat @@ -30,7 +38,9 @@ def self.add_to_archive(archive_io, child_path) exit_status = write_compressed_data(child_path, archive_io) if exit_status.success? - add_to_archive_commit(archive_io, child_path, start_pos, data_pos, stat) + add_to_archive_commit( + archive_io, child_path, start_pos, data_pos, stat + ) child_path.unlink else add_to_archive_rollback(archive_io, start_pos) @@ -99,6 +109,13 @@ def self.write_padding(size, io) io.write("\0" * remainder) end + # Archive the given dataset + # + # @param [IO] archive_io the IO of the target archive + # @param [Pathname] path path to the dataset folder + # @param [Boolean] full whether we're arching the complete dataset (true), + # or only the files that we know are not being written to (for log + # directories of running Syskit instances) def self.archive_dataset(archive_io, path, full:) puts "Archiving dataset #{path} in #{full ? "full" : "partial"} mode" candidates = path.enum_for(:each_entry).map { path / _1 } @@ -109,21 +126,42 @@ def self.archive_dataset(archive_io, path, full:) end end + # Filters all candidates for archiving to return the ones relevant for a + # partial archive (i.e. excluding files that are being written to) + # + # @param [Array] candidates + # @return [Array] files that should be archived def self.archive_partial_filter_candidates(candidates) - per_file_and_idx = candidates.each_with_object({}) do |path, h| + per_file_and_idx = filter_and_group_pocolog_files(candidates) + per_file_and_idx.each_value.flat_map do |logs| + logs.delete(logs.keys.max) + logs.values + end + end + + # Filter the pocolog files from the given candidates and sort them + # by basename and log index + # + # @param [Array] candidates + # @return [{String=>{Integer=>Pathname}}] + def self.filter_and_group_pocolog_files(candidates) + candidates.each_with_object({}) do |path, h| name = path.basename.to_s if (m = /\.(\d+)\.log$/.match(name)) per_file = (h[m.pre_match] ||= {}) per_file[Integer(m[1])] = path end end - - per_file_and_idx.each_value.flat_map do |logs| - logs.delete(logs.keys.max) - logs.values - end end + # Iterate over all datasets in a Roby log root folder and archive them + # + # The method assumes the last dataset is the current one (i.e. the running + # one), and will only archive already rotated files. + # + # @param [Pathname] root_dir the log root folder + # @param [Pathname] target_dir the folder in which to save the + # archived datasets def self.process_root_folder(root_dir, target_dir) candidates = find_all_dataset_folders(root_dir) running = candidates.last @@ -184,7 +222,7 @@ def self.split_name(name) # rubocop:disable Metrics/AbcSize newname = "#{nxt}/#{newname}" end - prefix = (parts + [nxt]).join('/') + prefix = (parts + [nxt]).join("/") name = newname end diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index 9f9b4be42..712f324c6 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -1,4 +1,3 @@ -#! /usr/bin/env ruby # frozen_string_literal: true require "pathname" diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 54d182041..37de8b54f 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -91,7 +91,7 @@ def assert_entry_matches(entry, data, name:, content:) it "adds a compressed version of the input I/O to the archive "\ "and deletes the input file" do - in_path = make_in_file "something.txt", "something" + something = make_in_file "something.txt", "something" @archive_path.open("w") do |archive_io| @in_files.each do |in_path| @@ -104,7 +104,7 @@ def assert_entry_matches(entry, data, name:, content:) assert_entry_matches( *entries.first, name: "something.txt.zst", content: "something" ) - refute in_path.exist? + refute something.exist? end it "creates a multi-file archive" do From c4919abf3b92941f60b2a79edb2abcce8ef2cdfd Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 18 Jul 2023 15:10:10 -0300 Subject: [PATCH 084/260] fix: ignore exceptions and rollback in LogRuntimeArchive.add_to_archive --- lib/syskit/cli/log_runtime_archive.rb | 6 +++-- test/cli/test_log_runtime_archive.rb | 32 ++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index e6ca1e509..47acc8487 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -42,13 +42,15 @@ def self.add_to_archive(archive_io, child_path) archive_io, child_path, start_pos, data_pos, stat ) child_path.unlink + true else add_to_archive_rollback(archive_io, start_pos) + false end - rescue Exception => e # rubocop:disable Lint/RescueException + Roby.display_exception(STDOUT, e) add_to_archive_rollback(archive_io, start_pos) if start_pos - raise + false end # Finalize appending a file in the archive diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 37de8b54f..c46484b1a 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -131,13 +131,39 @@ def assert_entry_matches(entry, data, name:, content:) bli = make_in_file "bli.txt", "bli" @archive_path.open("w") do |archive_io| - LogRuntimeArchive.add_to_archive(archive_io, bla) + assert LogRuntimeArchive.add_to_archive(archive_io, bla) FlexMock.use(Process) do |mock| mock.should_receive(:waitpid2).once .and_return([10, flexmock(success?: false)]) - LogRuntimeArchive.add_to_archive(archive_io, blo) + refute LogRuntimeArchive.add_to_archive(archive_io, blo) end - LogRuntimeArchive.add_to_archive(archive_io, bli) + assert LogRuntimeArchive.add_to_archive(archive_io, bli) + end + + entries = read_archive + assert_equal 2, entries.size + assert_entry_matches(*entries[0], name: "bla.txt.zst", content: "bla") + assert_entry_matches(*entries[1], name: "bli.txt.zst", content: "bli") + refute bla.exist? + assert blo.exist? + refute bli.exist? + end + + it "restores the file as it was and keeps the input file if an "\ + "exception occurs" do + bla = make_in_file "bla.txt", "bla" + blo = make_in_file "blo.txt", "blo" + bli = make_in_file "bli.txt", "bli" + + @archive_path.open("w") do |archive_io| + assert LogRuntimeArchive.add_to_archive(archive_io, bla) + FlexMock.use(Process) do |mock| + mock.should_receive(:waitpid2).once + .and_raise(Exception.new) + flexmock(Roby).should_receive(:display_exception).once + refute LogRuntimeArchive.add_to_archive(archive_io, blo) + end + assert LogRuntimeArchive.add_to_archive(archive_io, bli) end entries = read_archive From 77aaaea1803b9ea2c5dcd81cd040994f57d17e82 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 18 Jul 2023 15:14:25 -0300 Subject: [PATCH 085/260] fix: make reporting of log-runtime-archive configurable --- lib/syskit/cli/log_runtime_archive.rb | 34 ++++++++++++++++------- lib/syskit/scripts/log_runtime_archive.rb | 7 ++++- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 47acc8487..3cd54c3cc 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -28,8 +28,8 @@ def self.find_all_dataset_folders(root_dir) # Safely add an entry into an archive, compressing it with zstd # # @return [Boolean] true if the file was added successfully, false otherwise - def self.add_to_archive(archive_io, child_path) - puts "adding #{child_path}" + def self.add_to_archive(archive_io, child_path, logger: null_logger) + logger.info "adding #{child_path}" stat = child_path.stat start_pos = archive_io.tell @@ -44,12 +44,14 @@ def self.add_to_archive(archive_io, child_path) child_path.unlink true else - add_to_archive_rollback(archive_io, start_pos) + add_to_archive_rollback(archive_io, start_pos, logger: logger) false end rescue Exception => e # rubocop:disable Lint/RescueException Roby.display_exception(STDOUT, e) - add_to_archive_rollback(archive_io, start_pos) if start_pos + if start_pos + add_to_archive_rollback(archive_io, start_pos, logger: logger) + end false end @@ -67,7 +69,8 @@ def self.add_to_archive_commit( end # Revert the addition of a file in the archive, after an error - def self.add_to_archive_rollback(archive_io, start_pos) + def self.add_to_archive_rollback(archive_io, start_pos, logger:) + logger.warn "failed addition to archive, rolling back to known-good state" archive_io.truncate(start_pos) archive_io.seek(start_pos, IO::SEEK_SET) end @@ -111,6 +114,13 @@ def self.write_padding(size, io) io.write("\0" * remainder) end + # Create a logger that will display nothing + def self.null_logger + logger = Logger.new(STDOUT) + logger.level = Logger::FATAL + 1 + logger + end + # Archive the given dataset # # @param [IO] archive_io the IO of the target archive @@ -118,13 +128,15 @@ def self.write_padding(size, io) # @param [Boolean] full whether we're arching the complete dataset (true), # or only the files that we know are not being written to (for log # directories of running Syskit instances) - def self.archive_dataset(archive_io, path, full:) - puts "Archiving dataset #{path} in #{full ? "full" : "partial"} mode" + def self.archive_dataset(archive_io, path, full:, logger: null_logger) + logger.info( + "Archiving dataset #{path} in #{full ? 'full' : 'partial'} mode" + ) candidates = path.enum_for(:each_entry).map { path / _1 } candidates = archive_partial_filter_candidates(candidates) unless full candidates.each do |child_path| - add_to_archive(archive_io, child_path) + add_to_archive(archive_io, child_path, logger: logger) end end @@ -164,7 +176,7 @@ def self.filter_and_group_pocolog_files(candidates) # @param [Pathname] root_dir the log root folder # @param [Pathname] target_dir the folder in which to save the # archived datasets - def self.process_root_folder(root_dir, target_dir) + def self.process_root_folder(root_dir, target_dir, logger: null_logger) candidates = find_all_dataset_folders(root_dir) running = candidates.last candidates.each do |child| @@ -178,7 +190,9 @@ def self.process_root_folder(root_dir, target_dir) archive_path.open(mode) do |archive_io| archive_io.seek(0, IO::SEEK_END) - archive_dataset(archive_io, child, full: child != running) + archive_dataset( + archive_io, child, logger: logger, full: child != running + ) end end end diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index 712f324c6..a8f0d8443 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -21,8 +21,13 @@ POLLING_PERIOD = 600 +logger = Logger.new(STDOUT) +logger.level = Logger::INFO + loop do - Syskit::CLI::LogRuntimeArchive.process_root_folder(root_dir, target_dir) + Syskit::CLI::LogRuntimeArchive.process_root_folder( + root_dir, target_dir, logger: logger + ) puts "Archived pending logs, sleeping #{POLLING_PERIOD}s" sleep POLLING_PERIOD From c2abbff42773293b14acee3bfd8a40cdb9e1b879 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 18 Jul 2023 15:25:15 -0300 Subject: [PATCH 086/260] fix: test LogRuntimeArchive.archive_dataset --- lib/syskit/cli/log_runtime_archive.rb | 4 +- test/cli/test_log_runtime_archive.rb | 134 +++++++++++++++++--------- 2 files changed, 94 insertions(+), 44 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 3cd54c3cc..da3d6e9dc 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -132,7 +132,9 @@ def self.archive_dataset(archive_io, path, full:, logger: null_logger) logger.info( "Archiving dataset #{path} in #{full ? 'full' : 'partial'} mode" ) - candidates = path.enum_for(:each_entry).map { path / _1 } + candidates = + path.enum_for(:each_entry).map { path / _1 } + .find_all { _1.file? } candidates = archive_partial_filter_candidates(candidates) unless full candidates.each do |child_path| diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index c46484b1a..3cc91388c 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -6,18 +6,13 @@ module Syskit module CLI describe LogRuntimeArchive do - describe ".find_all_dataset_folders" do - before do - @root = make_tmppath - end - - def make_valid_folder(name) - path = (@root / name) - path.mkpath - FileUtils.touch(path / "info.yml") - path - end + before do + @root = make_tmppath + @archive_path = (make_tmppath / "archive.tar") + @in_files = [] + end + describe ".find_all_dataset_folders" do it "returns the directories that look like a dataset path" do path = make_valid_folder("20229523-1104.1") assert_equal( @@ -57,38 +52,6 @@ def make_valid_folder(name) end describe ".add_to_archive" do - before do - @root = make_tmppath - @archive_path = (make_tmppath / "archive.tar") - @in_files = [] - end - - def make_in_file(name, content) - path = (@root / name) - path.write(content) - @in_files << path - path - end - - def read_archive - tar = Archive::Tar::Minitar::Input.open(@archive_path.open("r")) - tar.each_entry.map { |e| [e, e.read] } - end - - def decompress_data(data) - IO.popen(["zstd", "-d", "--stdout"], "r+") do |io| - io.write data - io.close_write - io.read - end - end - - def assert_entry_matches(entry, data, name:, content:) - assert entry.file? - assert_equal name, entry.full_name - assert_equal content, decompress_data(data) - end - it "adds a compressed version of the input I/O to the archive "\ "and deletes the input file" do something = make_in_file "something.txt", "something" @@ -175,6 +138,91 @@ def assert_entry_matches(entry, data, name:, content:) refute bli.exist? end end + + describe ".archive_dataset" do + it "does a full archive of a given dataset" do + dataset = make_valid_folder("20220434-2023") + make_in_file "test.0.log", "test0", root: dataset + make_in_file "test.1.log", "test1", root: dataset + make_in_file "something.txt", "something", root: dataset + + @archive_path.open("w") do |archive_io| + flexmock(LogRuntimeArchive) + .should_receive(:add_to_archive).times(4).pass_thru + LogRuntimeArchive.archive_dataset(archive_io, dataset, full: true) + end + + entries = read_archive + assert_equal 4, entries.size + entries = entries.sort_by { _1[0].full_name } + assert_entry_matches( + *entries[0], name: "info.yml.zst", content: "" + ) + assert_entry_matches( + *entries[1], name: "something.txt.zst", content: "something" + ) + assert_entry_matches( + *entries[2], name: "test.0.log.zst", content: "test0" + ) + assert_entry_matches( + *entries[3], name: "test.1.log.zst", content: "test1" + ) + end + + it "does a partial archive of a given dataset" do + dataset = make_valid_folder("20220434-2023") + make_in_file "test.0.log", "test0", root: dataset + make_in_file "test.1.log", "test1", root: dataset + make_in_file "something.txt", "something", root: dataset + + @archive_path.open("w") do |archive_io| + flexmock(LogRuntimeArchive) + .should_receive(:add_to_archive).times(1).pass_thru + LogRuntimeArchive.archive_dataset( + archive_io, dataset, full: false + ) + end + + entries = read_archive + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "test.0.log.zst", content: "test0" + ) + end + end + + def make_valid_folder(name) + path = (@root / name) + path.mkpath + FileUtils.touch(path / "info.yml") + path + end + + def make_in_file(name, content, root: @root) + path = (root / name) + path.write(content) + @in_files << path + path + end + + def read_archive + tar = Archive::Tar::Minitar::Input.open(@archive_path.open("r")) + tar.each_entry.map { |e| [e, e.read] } + end + + def decompress_data(data) + IO.popen(["zstd", "-d", "--stdout"], "r+") do |io| + io.write data + io.close_write + io.read + end + end + + def assert_entry_matches(entry, data, name:, content:) + assert entry.file? + assert_equal name, entry.full_name + assert_equal content, decompress_data(data) + end end end end From 68f24f430f8a67b39aadcb7540bd1b32bc763a70 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 18 Jul 2023 15:33:01 -0300 Subject: [PATCH 087/260] fix: test LogRuntimeArchive.process_root_folder --- test/cli/test_log_runtime_archive.rb | 61 +++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 3cc91388c..3350f42e9 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -191,6 +191,63 @@ module CLI end end + describe ".process_root_folder" do + it "archives all folders, the last one only partially" do + dataset0 = make_valid_folder("20220434-2023") + dataset1 = make_valid_folder("20220434-2024") + dataset2 = make_valid_folder("20220434-2025") + + archive_dir = make_tmppath + flexmock(LogRuntimeArchive) + .should_receive(:archive_dataset) + .with( + ->(p) { p.path == (archive_dir / "20220434-2023.tar").to_s }, + dataset0, + hsh(full: true) + ).once.pass_thru + flexmock(LogRuntimeArchive) + .should_receive(:archive_dataset) + .with( + ->(p) { p.path == (archive_dir / "20220434-2024.tar").to_s }, + dataset1, + hsh(full: true) + ).once.pass_thru + flexmock(LogRuntimeArchive) + .should_receive(:archive_dataset) + .with( + ->(p) { p.path == (archive_dir / "20220434-2025.tar").to_s }, + dataset2, + hsh(full: false) + ).once.pass_thru + + LogRuntimeArchive.process_root_folder(@root, archive_dir) + + assert (archive_dir / "20220434-2023.tar").file? + assert (archive_dir / "20220434-2024.tar").file? + assert (archive_dir / "20220434-2025.tar").file? + end + + it "appends to existing archives" do + dataset = make_valid_folder("20220434-2023") + make_in_file "test.0.log", "test0", root: dataset + make_in_file "test.1.log", "test1", root: dataset + + archive_dir = make_tmppath + LogRuntimeArchive.process_root_folder(@root, archive_dir) + make_in_file "test.2.log", "test2", root: dataset + LogRuntimeArchive.process_root_folder(@root, archive_dir) + + entries = read_archive(path: archive_dir / "20220434-2023.tar") + assert_equal 2, entries.size + assert_entry_matches( + *entries[0], name: "test.0.log.zst", content: "test0" + ) + assert_entry_matches( + *entries[1], name: "test.1.log.zst", content: "test1" + ) + end + end + def make_valid_folder(name) path = (@root / name) path.mkpath @@ -205,8 +262,8 @@ def make_in_file(name, content, root: @root) path end - def read_archive - tar = Archive::Tar::Minitar::Input.open(@archive_path.open("r")) + def read_archive(path: @archive_path) + tar = Archive::Tar::Minitar::Input.open(path.open("r")) tar.each_entry.map { |e| [e, e.read] } end From e6bcccc0009347477dad254d476c75f171ce11b5 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 18 Jul 2023 16:46:59 -0300 Subject: [PATCH 088/260] fix: add depndencies for log_runtime_archive --- manifest.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manifest.xml b/manifest.xml index f8337eb32..63d56ba9f 100644 --- a/manifest.xml +++ b/manifest.xml @@ -23,6 +23,10 @@ + + + + From 6cdc805ba7fb1b8534ce23ece7cf1627a9fa7bad Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 20 Jul 2023 17:45:50 -0300 Subject: [PATCH 089/260] feat: implement splitting the archives at a given size 10G by default, to ease management and copying. --- lib/syskit/cli/log_runtime_archive.rb | 142 ++++++++++++++++++++------ test/cli/test_log_runtime_archive.rb | 140 +++++++++++++++++++------ 2 files changed, 219 insertions(+), 63 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index da3d6e9dc..f82f17708 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -10,7 +10,103 @@ module CLI # compressing files using zstd # # It depends on the syskit instance using log rotation - module LogRuntimeArchive + class LogRuntimeArchive + DEFAULT_MAX_ARCHIVE_SIZE = 10_000_000_000 # 10G + + def initialize( + root_dir, target_dir, + logger: LogRuntimeArchive.null_logger, + max_archive_size: DEFAULT_MAX_ARCHIVE_SIZE + ) + @last_archive_index = {} + @logger = logger + @root_dir = root_dir + @target_dir = target_dir + @max_archive_size = max_archive_size + end + + # Iterate over all datasets in a Roby log root folder and archive them + # + # The method assumes the last dataset is the current one (i.e. the running + # one), and will only archive already rotated files. + # + # @param [Pathname] root_dir the log root folder + # @param [Pathname] target_dir the folder in which to save the + # archived datasets + def process_root_folder + candidates = self.class.find_all_dataset_folders(@root_dir) + running = candidates.last + candidates.each do |child| + process_dataset(child, full: child != running) + end + end + + def process_dataset(child, full:) + use_existing = true + loop do + open_archive_for( + child.basename.to_s, use_existing: use_existing + ) do |io| + if io.tell > @max_archive_size + use_existing = false + break + end + + dataset_complete = self.class.archive_dataset( + io, child, + logger: @logger, full: full, + max_size: @max_archive_size + ) + return if dataset_complete + end + + use_existing = false + end + end + + # Create or open an archive + # + # The method will find an archive to open or create, do it and + # yield the corresponding IO. The archives are named #{basename}.${INDEX}.tar + # + # @param [Boolean] use_existing if false, always create a new + # archive. If true, reuse the last archive that was created, if + # present, or create ${basename}.0.tar otherwise. + def open_archive_for(basename, use_existing: true) + last_index = find_last_archive_index(basename) + + index, mode = + if !last_index + [0, "w"] + elsif use_existing + [last_index, "r+"] + else + [last_index + 1, "w"] + end + + archive_path = @target_dir / "#{basename}.#{index}.tar" + archive_path.open(mode) do |io| + io.seek(0, IO::SEEK_END) + yield(io) + end + end + + # Find the last archive index used for a given basename + # + # @param [String] basename the archive basename + def find_last_archive_index(basename) + i = @last_archive_index[basename] || 0 + last_i = nil + loop do + candidate = @target_dir / "#{basename}.#{i}.tar" + return last_i unless candidate.exist? + + @last_archive_index[basename] = last_i + last_i = i + i += 1 + end + end + # Find all dataset-looking folders within a root log folder def self.find_all_dataset_folders(root_dir) candidates = root_dir.enum_for(:each_entry).map do |child| @@ -128,7 +224,13 @@ def self.null_logger # @param [Boolean] full whether we're arching the complete dataset (true), # or only the files that we know are not being written to (for log # directories of running Syskit instances) - def self.archive_dataset(archive_io, path, full:, logger: null_logger) + # @return [Boolean] true if we're done processing this dataset. False + # if processing was interrupted by e.g. an archive that reached the + # max_archive_size limit + def self.archive_dataset( + archive_io, path, + full:, logger: null_logger, max_size: DEFAULT_MAX_ARCHIVE_SIZE + ) logger.info( "Archiving dataset #{path} in #{full ? 'full' : 'partial'} mode" ) @@ -136,10 +238,13 @@ def self.archive_dataset(archive_io, path, full:, logger: null_logger) path.enum_for(:each_entry).map { path / _1 } .find_all { _1.file? } candidates = archive_partial_filter_candidates(candidates) unless full - - candidates.each do |child_path| + candidates.each_with_index do |child_path, i| add_to_archive(archive_io, child_path, logger: logger) + + return (i == candidates.size - 1) if archive_io.tell > max_size end + + true end # Filters all candidates for archiving to return the ones relevant for a @@ -170,35 +275,6 @@ def self.filter_and_group_pocolog_files(candidates) end end - # Iterate over all datasets in a Roby log root folder and archive them - # - # The method assumes the last dataset is the current one (i.e. the running - # one), and will only archive already rotated files. - # - # @param [Pathname] root_dir the log root folder - # @param [Pathname] target_dir the folder in which to save the - # archived datasets - def self.process_root_folder(root_dir, target_dir, logger: null_logger) - candidates = find_all_dataset_folders(root_dir) - running = candidates.last - candidates.each do |child| - archive_path = target_dir / "#{child.basename}.tar" - mode = - if archive_path.exist? - "r+" - else - "w" - end - - archive_path.open(mode) do |archive_io| - archive_io.seek(0, IO::SEEK_END) - archive_dataset( - archive_io, child, logger: logger, full: child != running - ) - end - end - end - extend Archive::Tar::Minitar::ByteSize # Copy a tar header (copied from minitar) diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 3350f42e9..f7a321c9d 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -192,39 +192,111 @@ module CLI end describe ".process_root_folder" do + before do + @archive_dir = make_tmppath + @process = LogRuntimeArchive.new(@root, @archive_dir) + end + it "archives all folders, the last one only partially" do dataset0 = make_valid_folder("20220434-2023") dataset1 = make_valid_folder("20220434-2024") dataset2 = make_valid_folder("20220434-2025") - archive_dir = make_tmppath - flexmock(LogRuntimeArchive) - .should_receive(:archive_dataset) - .with( - ->(p) { p.path == (archive_dir / "20220434-2023.tar").to_s }, - dataset0, - hsh(full: true) - ).once.pass_thru - flexmock(LogRuntimeArchive) - .should_receive(:archive_dataset) - .with( - ->(p) { p.path == (archive_dir / "20220434-2024.tar").to_s }, - dataset1, - hsh(full: true) - ).once.pass_thru - flexmock(LogRuntimeArchive) - .should_receive(:archive_dataset) - .with( - ->(p) { p.path == (archive_dir / "20220434-2025.tar").to_s }, - dataset2, - hsh(full: false) - ).once.pass_thru + should_archive_dataset(dataset0, "20220434-2023.0.tar", full: true) + should_archive_dataset(dataset1, "20220434-2024.0.tar", full: true) + should_archive_dataset(dataset2, "20220434-2025.0.tar", full: false) + @process.process_root_folder + + assert (@archive_dir / "20220434-2023.0.tar").file? + assert (@archive_dir / "20220434-2024.0.tar").file? + assert (@archive_dir / "20220434-2025.0.tar").file? + end + + it "splits the archive according to the max size" do + dataset = make_valid_folder("20220434-2023") + (dataset / "test.0.log") + .write(test0 = Base64.encode64(Random.bytes(1024))) + (dataset / "test.1.log") + .write(test1 = Base64.encode64(Random.bytes(1024))) + (dataset / "test.2.log").write(Base64.encode64(Random.bytes(1024))) + process = LogRuntimeArchive.new( + @root, @archive_dir, max_archive_size: 1024 + ) + process.process_root_folder + + entries = read_archive(path: @archive_dir / "20220434-2023.0.tar") + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "test.0.log.zst", content: test0 + ) + + entries = read_archive(path: @archive_dir / "20220434-2023.1.tar") + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "test.1.log.zst", content: test1 + ) + + refute (@archive_dir / "20220434-2023.2.tar").exist? + end - LogRuntimeArchive.process_root_folder(@root, archive_dir) + it "appends to the last created archive" do + dataset = make_valid_folder("20220434-2023") + (dataset / "test.0.log") + .write(Base64.encode64(Random.bytes(1024))) + (dataset / "test.1.log") + .write(test1 = Base64.encode64(Random.bytes(128))) + (dataset / "test.2.log") + .write(test2 = Base64.encode64(Random.bytes(128))) + process = LogRuntimeArchive.new( + @root, @archive_dir, max_archive_size: 1024 + ) + process.process_root_folder - assert (archive_dir / "20220434-2023.tar").file? - assert (archive_dir / "20220434-2024.tar").file? - assert (archive_dir / "20220434-2025.tar").file? + (dataset / "test.3.log").write(Base64.encode64(Random.bytes(1024))) + process.process_root_folder + + entries = read_archive(path: @archive_dir / "20220434-2023.1.tar") + assert_equal 2, entries.size + assert_entry_matches( + *entries[0], name: "test.1.log.zst", content: test1 + ) + assert_entry_matches( + *entries[1], name: "test.2.log.zst", content: test2 + ) + + refute (@archive_dir / "20220434-2023.2.tar").exist? + end + + it "creates a new archive if the last archive is already "\ + "above the limit" do + dataset = make_valid_folder("20220434-2023") + (dataset / "test.0.log") + .write(Base64.encode64(Random.bytes(1024))) + (dataset / "test.1.log") + .write(test1 = Base64.encode64(Random.bytes(1024))) + (dataset / "test.2.log") + .write(test2 = Base64.encode64(Random.bytes(1024))) + process = LogRuntimeArchive.new( + @root, @archive_dir, max_archive_size: 1024 + ) + process.process_root_folder + + (dataset / "test.3.log").write(Base64.encode64(Random.bytes(1024))) + process.process_root_folder + + entries = read_archive(path: @archive_dir / "20220434-2023.1.tar") + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "test.1.log.zst", content: test1 + ) + + entries = read_archive(path: @archive_dir / "20220434-2023.2.tar") + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "test.2.log.zst", content: test2 + ) + + refute (@archive_dir / "20220434-2023.3.tar").exist? end it "appends to existing archives" do @@ -232,12 +304,11 @@ module CLI make_in_file "test.0.log", "test0", root: dataset make_in_file "test.1.log", "test1", root: dataset - archive_dir = make_tmppath - LogRuntimeArchive.process_root_folder(@root, archive_dir) + @process.process_root_folder make_in_file "test.2.log", "test2", root: dataset - LogRuntimeArchive.process_root_folder(@root, archive_dir) + @process.process_root_folder - entries = read_archive(path: archive_dir / "20220434-2023.tar") + entries = read_archive(path: @archive_dir / "20220434-2023.0.tar") assert_equal 2, entries.size assert_entry_matches( *entries[0], name: "test.0.log.zst", content: "test0" @@ -246,6 +317,15 @@ module CLI *entries[1], name: "test.1.log.zst", content: "test1" ) end + + def should_archive_dataset(dataset, archive_basename, full:) + flexmock(LogRuntimeArchive) + .should_receive(:archive_dataset) + .with( + ->(p) { p.path == (@archive_dir / archive_basename).to_s }, + dataset, hsh(full: full) + ).once.pass_thru + end end def make_valid_folder(name) From 6e43da51986e4d0f0b9149b1b7e89c58e56c6713 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Fri, 21 Jul 2023 11:04:34 -0300 Subject: [PATCH 090/260] feat: gather all non-rotated logs in the last archive(s) This will make it simpler to delete non-relevant logs mistakenly saved. The last tarball(s) will contain the logs that are for a whole execution, while the previous ones are "slices" in time of the data. --- lib/syskit/cli/log_runtime_archive.rb | 67 +++++++-- test/cli/test_log_runtime_archive.rb | 193 +++++++++++++++++++++++--- 2 files changed, 233 insertions(+), 27 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index f82f17708..2d1121e76 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -234,30 +234,75 @@ def self.archive_dataset( logger.info( "Archiving dataset #{path} in #{full ? 'full' : 'partial'} mode" ) - candidates = - path.enum_for(:each_entry).map { path / _1 } - .find_all { _1.file? } - candidates = archive_partial_filter_candidates(candidates) unless full + candidates = each_file_from_path(path).to_a + complete, candidates = + if full + archive_filter_candidates_full(candidates) + else + archive_filter_candidates_partial(candidates) + end + candidates.each_with_index do |child_path, i| add_to_archive(archive_io, child_path, logger: logger) - return (i == candidates.size - 1) if archive_io.tell > max_size + if archive_io.tell > max_size + return (complete && (i == candidates.size - 1)) + end end - true + complete + end + + # Enumerate the children of a path that are files + # + # @yieldparam [Pathname] file_path the full path to the file + def self.each_file_from_path(path) + return enum_for(:each_file_from_path, path) unless block_given? + + path.each_entry do |child_path| + full = path / child_path + yield(full) if full.file? + end end - # Filters all candidates for archiving to return the ones relevant for a - # partial archive (i.e. excluding files that are being written to) + # Filters all candidates for archiving to return the ones that should + # be archived in `full` mode at this point in time + # + # The method either returns the remaining rotated logs, or if there + # are none, the non-rotated files. This ensures that the archiver groups + # all non-rotated files in a single archive. # # @param [Array] candidates - # @return [Array] files that should be archived - def self.archive_partial_filter_candidates(candidates) + # @return [(Boolean,Array)] a flag that tell whether the candidate + # array is complete or not, and the files that should be archived. If + # the flag is true, the assumption is that after having archived the files + # that were returned, the archiving loop should try archiving again. + def self.archive_filter_candidates_full(candidates) per_file_and_idx = filter_and_group_pocolog_files(candidates) - per_file_and_idx.each_value.flat_map do |logs| + rotated_logs = per_file_and_idx.each_value.flat_map(&:values) + unless rotated_logs.empty? + return [(candidates - rotated_logs).empty?, rotated_logs] + end + + [true, candidates] + end + + # Filters all candidates for archiving to return the ones that should + # be archived in `partial` mode at this point in time + # + # The method returns the rotated logs that are known to be complete. + # + # @param [Array] candidates + # @return [(true,Array)] files that should be archived. The + # boolean is here for consistency with {.archive_filter_candidates_full} + def self.archive_filter_candidates_partial(candidates) + per_file_and_idx = filter_and_group_pocolog_files(candidates) + complete_log_files = per_file_and_idx.each_value.flat_map do |logs| logs.delete(logs.keys.max) logs.values end + + [true, complete_log_files] end # Filter the pocolog files from the given candidates and sort them diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index f7a321c9d..8a29c1228 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -140,32 +140,50 @@ module CLI end describe ".archive_dataset" do - it "does a full archive of a given dataset" do + it "in full mode, archives only the rotated logs if there are some" do dataset = make_valid_folder("20220434-2023") make_in_file "test.0.log", "test0", root: dataset make_in_file "test.1.log", "test1", root: dataset make_in_file "something.txt", "something", root: dataset - @archive_path.open("w") do |archive_io| + ret = @archive_path.open("w") do |archive_io| flexmock(LogRuntimeArchive) - .should_receive(:add_to_archive).times(4).pass_thru + .should_receive(:add_to_archive).times(2).pass_thru LogRuntimeArchive.archive_dataset(archive_io, dataset, full: true) end + refute ret entries = read_archive - assert_equal 4, entries.size + assert_equal 2, entries.size entries = entries.sort_by { _1[0].full_name } assert_entry_matches( - *entries[0], name: "info.yml.zst", content: "" + *entries[0], name: "test.0.log.zst", content: "test0" ) assert_entry_matches( - *entries[1], name: "something.txt.zst", content: "something" + *entries[1], name: "test.1.log.zst", content: "test1" ) + end + + it "in full mode, archives the non-rotated logs "\ + "if there are no rotated logs" do + dataset = make_valid_folder("20220434-2023") + make_in_file "something.txt", "something", root: dataset + + ret = @archive_path.open("w") do |archive_io| + flexmock(LogRuntimeArchive) + .should_receive(:add_to_archive).times(2).pass_thru + LogRuntimeArchive.archive_dataset(archive_io, dataset, full: true) + end + assert ret + + entries = read_archive + assert_equal 2, entries.size + entries = entries.sort_by { _1[0].full_name } assert_entry_matches( - *entries[2], name: "test.0.log.zst", content: "test0" + *entries[0], name: "info.yml.zst", content: "" ) assert_entry_matches( - *entries[3], name: "test.1.log.zst", content: "test1" + *entries[1], name: "something.txt.zst", content: "something" ) end @@ -175,13 +193,133 @@ module CLI make_in_file "test.1.log", "test1", root: dataset make_in_file "something.txt", "something", root: dataset - @archive_path.open("w") do |archive_io| + ret = @archive_path.open("w") do |archive_io| flexmock(LogRuntimeArchive) .should_receive(:add_to_archive).times(1).pass_thru LogRuntimeArchive.archive_dataset( archive_io, dataset, full: false ) end + assert ret + + entries = read_archive + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "test.0.log.zst", content: "test0" + ) + end + + it "stops processing when it reaches the max size" do + dataset = make_valid_folder("20220434-2023") + make_in_file "test.0.log", "test0", root: dataset + make_in_file "test.1.log", "test1", root: dataset + make_in_file "test.2.log", "test2", root: dataset + make_in_file "something.txt", "something", root: dataset + + ret = @archive_path.open("w") do |archive_io| + flexmock(LogRuntimeArchive) + .should_receive(:add_to_archive).times(1).pass_thru + LogRuntimeArchive.archive_dataset( + archive_io, dataset, full: false, max_size: 4 + ) + end + refute ret + + entries = read_archive + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "test.0.log.zst", content: "test0" + ) + end + + it "always adds at least a file, "\ + "regardless of the current size of the archive" do + dataset = make_valid_folder("20220434-2023") + make_in_file "test.0.log", "test0", root: dataset + make_in_file "test.1.log", "test1", root: dataset + make_in_file "test.2.log", "test2", root: dataset + make_in_file "something.txt", "something", root: dataset + + ret = @archive_path.open("w") do |archive_io| + flexmock(LogRuntimeArchive) + .should_receive(:add_to_archive).times(2).pass_thru + LogRuntimeArchive.archive_dataset( + archive_io, dataset, full: false, max_size: 4 + ) + LogRuntimeArchive.archive_dataset( + archive_io, dataset, full: false, max_size: 4 + ) + end + assert ret + + entries = read_archive + assert_equal 2, entries.size + assert_entry_matches( + *entries[0], name: "test.0.log.zst", content: "test0" + ) + assert_entry_matches( + *entries[1], name: "test.1.log.zst", content: "test1" + ) + end + + it "reports a complete processing in full mode if the size limit "\ + "is reached on the last rotated log and "\ + "there are no non-rotated logs" do + dataset = @root / "20220434-2023" + dataset.mkpath + make_in_file "test.0.log", "test0", root: dataset + + ret = @archive_path.open("w") do |archive_io| + flexmock(LogRuntimeArchive) + .should_receive(:add_to_archive).times(1).pass_thru + LogRuntimeArchive.archive_dataset( + archive_io, dataset, full: true, max_size: 0 + ) + end + assert ret + + entries = read_archive + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "test.0.log.zst", content: "test0" + ) + end + + it "reports a complete processing in full mode if the size limit "\ + "is reached on the last non-rotated log and "\ + "there are no rotated logs" do + dataset = make_valid_folder("20220434-2023") + + ret = @archive_path.open("w") do |archive_io| + flexmock(LogRuntimeArchive) + .should_receive(:add_to_archive).times(1).pass_thru + LogRuntimeArchive.archive_dataset( + archive_io, dataset, full: true, max_size: 0 + ) + end + assert ret + + entries = read_archive + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "info.yml.zst", content: "" + ) + end + + it "reports a complete processing in partial mode if the size limit "\ + "is reached on the last log to process" do + dataset = make_valid_folder("20220434-2023") + make_in_file "test.0.log", "test0", root: dataset + make_in_file "test.1.log", "test1", root: dataset + + ret = @archive_path.open("w") do |archive_io| + flexmock(LogRuntimeArchive) + .should_receive(:add_to_archive).times(1).pass_thru + LogRuntimeArchive.archive_dataset( + archive_io, dataset, full: false, max_size: 512 + ) + end + assert ret entries = read_archive assert_equal 1, entries.size @@ -270,18 +408,15 @@ module CLI it "creates a new archive if the last archive is already "\ "above the limit" do dataset = make_valid_folder("20220434-2023") - (dataset / "test.0.log") - .write(Base64.encode64(Random.bytes(1024))) - (dataset / "test.1.log") - .write(test1 = Base64.encode64(Random.bytes(1024))) - (dataset / "test.2.log") - .write(test2 = Base64.encode64(Random.bytes(1024))) + make_random_file "test.0.log", root: dataset + test1 = make_random_file "test.1.log", root: dataset + test2 = make_random_file "test.2.log", root: dataset process = LogRuntimeArchive.new( @root, @archive_dir, max_archive_size: 1024 ) process.process_root_folder - (dataset / "test.3.log").write(Base64.encode64(Random.bytes(1024))) + make_random_file "test.3.log", root: dataset process.process_root_folder entries = read_archive(path: @archive_dir / "20220434-2023.1.tar") @@ -318,6 +453,26 @@ module CLI ) end + it "gathers all non-rotated logs in the very last archive" do + dataset = make_valid_folder("20220434-2023") + make_valid_folder("20220434-2024") + make_random_file "test.0.log", root: dataset + make_random_file "test.1.log", root: dataset + make_random_file "test.2.log", root: dataset + make_random_file "test.txt", root: dataset + make_random_file "test-PID.txt", root: dataset + + @process.process_root_folder + + entries = read_archive(path: @archive_dir / "20220434-2023.0.tar") + assert_equal %w[test.0.log.zst test.1.log.zst test.2.log.zst], + entries.map { _1.first.name }.sort + + entries = read_archive(path: @archive_dir / "20220434-2023.1.tar") + assert_equal %w[info.yml.zst test-PID.txt.zst test.txt.zst], + entries.map { _1.first.name }.sort + end + def should_archive_dataset(dataset, archive_basename, full:) flexmock(LogRuntimeArchive) .should_receive(:archive_dataset) @@ -335,6 +490,12 @@ def make_valid_folder(name) path end + def make_random_file(name, root: @root, size: 1024) + content = Base64.encode64(Random.bytes(size)) + make_in_file name, content, root: root + content + end + def make_in_file(name, content, root: @root) path = (root / name) path.write(content) From fef489016416df9d37b6c04271cd7129412d8a54 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Fri, 21 Jul 2023 11:23:24 -0300 Subject: [PATCH 091/260] fix: update CLI to pass parameters for log_runtime_archive on the command line --- lib/syskit/scripts/log_runtime_archive.rb | 74 ++++++++++++++--------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index a8f0d8443..474f6b281 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -1,34 +1,54 @@ # frozen_string_literal: true +# NOTE: this is NOT integrated in the Thor-based CLI to make it more independent +# (i.e. not depending on actually having Syskit installed) + require "pathname" +require "thor" require "syskit/cli/log_runtime_archive" -unless ARGV.size == 2 - STDERR.puts "usage: log_runtime_archive ROOT_DIR TARGET_DIR" - exit 1 -end - -root_dir = Pathname.new(ARGV[0]) -target_dir = Pathname.new(ARGV[1]) - -if !root_dir.directory? - warn "#{root_dir} is not a directory" - exit 1 -elsif !target_dir.directory? - warn "#{target_dir} is not a directory" - exit 1 +# Command-line definition for the log-runtime-archive syskit subcommand +class CLI < Thor + def self.exit_on_failure? + true + end + + desc "watch", "watch a dataset root folder and archive the datasets" + option :period, + type: :numeric, default: 600, desc: "polling period in seconds" + option :max_size, + type: :numeric, default: 10_000, desc: "max log size in MB" + default_task def watch(root_dir, target_dir) + root_dir = validate_directory_exists(root_dir) + target_dir = validate_directory_exists(target_dir) + archiver = make_archiver(root_dir, target_dir) + loop do + archiver.process_root_folder + + puts "Archived pending logs, sleeping #{options[:period]}s" + sleep options[:period] + end + end + + no_commands do + def validate_directory_exists(dir) + dir = Pathname.new(dir) + unless dir.directory? + raise ArgumentError, "#{dir} does not exist, or is not a directory" + end + + dir + end + + def make_archiver(root_dir, target_dir) + logger = Logger.new(STDOUT) + + Syskit::CLI::LogRuntimeArchive.new( + root_dir, target_dir, + logger: logger, max_archive_size: options[:max_size] * 1024**2 + ) + end + end end -POLLING_PERIOD = 600 - -logger = Logger.new(STDOUT) -logger.level = Logger::INFO - -loop do - Syskit::CLI::LogRuntimeArchive.process_root_folder( - root_dir, target_dir, logger: logger - ) - - puts "Archived pending logs, sleeping #{POLLING_PERIOD}s" - sleep POLLING_PERIOD -end +CLI.start(ARGV) From 21355b92a920a5dfc6a4be9256e9d4be952c41de Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 7 Aug 2023 12:34:18 -0300 Subject: [PATCH 092/260] fix: heisenbug in remote process server's #reap_dead_subprocesses We sometimes we get a PID that has no corresponding registered process and handlee_dead_subprocess returns nil. Do not notify the client about these processes as it possibly can't do anything about them. This was causing an exception in announce_dead_processes as it could not handle the nil. The reason for this process ID not being known is unknown right now. --- lib/syskit/roby_app/remote_processes/server.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 60c38cf5f..19654e508 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -254,7 +254,9 @@ def try_wait_for_subprocess_exit(pid) def reap_dead_subprocesses dead_processes = [] while (exited = try_wait_pid(-1)) - dead_processes << handle_dead_subprocess(*exited) + if (process = handle_dead_subprocess(*exited)) + dead_processes << process + end end dead_processes rescue Errno::ECHILD From e8e16842fe1d5b27fc0830fd415e2d935cb2427a Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 7 Aug 2023 12:35:23 -0300 Subject: [PATCH 093/260] fix: log when we get a PID that has no corresponding process (and which PID) As an attempt to understand the problem from the previous commit --- lib/syskit/roby_app/remote_processes/server.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/syskit/roby_app/remote_processes/server.rb b/lib/syskit/roby_app/remote_processes/server.rb index 19654e508..b5f75bd1a 100644 --- a/lib/syskit/roby_app/remote_processes/server.rb +++ b/lib/syskit/roby_app/remote_processes/server.rb @@ -281,7 +281,10 @@ def try_wait_pid(pid) def handle_dead_subprocess(exit_pid, exit_status) process_name, process = processes.find { |_, p| p.pid == exit_pid } - return unless process_name + unless process_name + warn "wait2 returned PID #{exit_pid}, which is not known" + return + end process.dead!(exit_status) processes.delete(process_name) From 50f51cf94e3ed98faa74ba97d988700857c8136b Mon Sep 17 00:00:00 2001 From: Wellington Castro Date: Wed, 30 Aug 2023 10:51:13 -0300 Subject: [PATCH 094/260] fix: handle selection of a job that does not have a planning task --- lib/syskit/gui/runtime_state.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/syskit/gui/runtime_state.rb b/lib/syskit/gui/runtime_state.rb index e03af281b..8c2178d9b 100644 --- a/lib/syskit/gui/runtime_state.rb +++ b/lib/syskit/gui/runtime_state.rb @@ -335,7 +335,7 @@ def update_tasks_info .first return unless job_task - placeholder_task = job_task.planned_task + placeholder_task = job_task.planned_task || job_task return unless placeholder_task dependency = placeholder_task.relation_graph_for(Roby::TaskStructure::Dependency) @@ -368,9 +368,8 @@ def update_tasks_info all_tasks.merge(tasks) tasks.each do |job| if job.kind_of?(Roby::Interface::Job) - if placeholder_task = job.planned_task - all_job_info[placeholder_task] = job - end + placeholder_task = job.planned_task || job + all_job_info[placeholder_task] = job end end update_orocos_tasks From 3f82c703ce04a53c14e431a064770f27db2ce3ee Mon Sep 17 00:00:00 2001 From: Wellington Castro Date: Wed, 30 Aug 2023 10:51:58 -0300 Subject: [PATCH 095/260] fix: display the job name instead of the action model name in the job list Roby's JobMonitor defaults to the action model name if no explicit job name was given, but allows us to set a job name programmatically when it makes sense. --- lib/syskit/gui/job_status_display.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syskit/gui/job_status_display.rb b/lib/syskit/gui/job_status_display.rb index ea8817514..8d02d620f 100644 --- a/lib/syskit/gui/job_status_display.rb +++ b/lib/syskit/gui/job_status_display.rb @@ -39,7 +39,7 @@ def initialize(job, batch_manager) ].freeze def label - "##{job.job_id} #{job.action_name}" + "##{job.job_id} #{job.job_name}" end def create_ui From 8d14fb29b9dd8982036fc4cf22ab85c435033751 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 13 Sep 2023 16:57:59 -0300 Subject: [PATCH 096/260] fix: process logs in order of their index while archiving --- lib/syskit/cli/log_runtime_archive.rb | 10 ++++++---- test/cli/test_log_runtime_archive.rb | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 2d1121e76..a4d058589 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -296,10 +296,12 @@ def self.archive_filter_candidates_full(candidates) # @return [(true,Array)] files that should be archived. The # boolean is here for consistency with {.archive_filter_candidates_full} def self.archive_filter_candidates_partial(candidates) - per_file_and_idx = filter_and_group_pocolog_files(candidates) - complete_log_files = per_file_and_idx.each_value.flat_map do |logs| - logs.delete(logs.keys.max) - logs.values + per_file_and_idx = + filter_and_group_pocolog_files(candidates) + .each_value { |logs| logs.delete(logs.keys.max) } + + complete_log_files = per_file_and_idx.flat_map do |_, logs| + logs.keys.sort.map { |i| logs[i] } end [true, complete_log_files] diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 8a29c1228..239c2dd67 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -232,6 +232,33 @@ module CLI ) end + it "orders log files according to their index" do + dataset = make_valid_folder("20220434-2023") + make_in_file "test.0.log", "test0", root: dataset + make_in_file "test.1.log", "test1", root: dataset + make_in_file "test.2.log", "test2", root: dataset + make_in_file "something.txt", "something", root: dataset + + ret = @archive_path.open("w") do |archive_io| + flexmock(LogRuntimeArchive) + .should_receive(:add_to_archive).times(1).pass_thru + flexmock(LogRuntimeArchive) + .should_receive(:each_file_from_path) + .pass_thru { |files| files.to_a.shuffle } + + LogRuntimeArchive.archive_dataset( + archive_io, dataset, full: false, max_size: 4 + ) + end + refute ret + + entries = read_archive + assert_equal 1, entries.size + assert_entry_matches( + *entries[0], name: "test.0.log.zst", content: "test0" + ) + end + it "always adds at least a file, "\ "regardless of the current size of the archive" do dataset = make_valid_folder("20220434-2023") From 4062c29a3e21fb761da0c44292f169a5210e1cde Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 18 Sep 2023 21:09:56 -0300 Subject: [PATCH 097/260] fix: handle log transfer setup and cleanup in prepare and shutdown These should only be configured if we're running the system, not just during setup/cleanup. --- lib/syskit/roby_app/plugin.rb | 9 ++++----- test/roby_app/test_plugin.rb | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index 692b5316b..fedcb4578 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -161,11 +161,9 @@ def self.setup(app) rtt_core_model = app.default_loader.task_model_from_name("RTT::TaskContext") Syskit::TaskContext.define_from_orogen(rtt_core_model, register: true) - - app.syskit_log_transfer_setup end - def syskit_log_transfer_setup + def syskit_log_transfer_prepare return unless Syskit.conf.log_transfer.ip unless Syskit.conf.log_rotation_period @@ -178,7 +176,7 @@ def syskit_log_transfer_setup @syskit_log_transfer_manager = LogTransferManager.new(conf) end - def syskit_log_transfer_cleanup + def syskit_log_transfer_shutdown syskit_log_transfer_manager&.dispose(syskit_log_transfer_process_servers) @syskit_log_transfer_manager = nil end @@ -253,13 +251,13 @@ def self.cleanup(app) app.syskit_remove_configuration_changes_listener end - app.syskit_log_transfer_cleanup stop_local_process_server(app) disconnect_all_process_servers end # Hook called by the main application to prepare for execution def self.prepare(app) + app.syskit_log_transfer_prepare @handler_ids = plug_engine_in_roby(app.execution_engine) if Syskit.conf.log_rotation_period @@ -282,6 +280,7 @@ def self.shutdown(app) unplug_engine_from_roby(@handler_ids.values, app.execution_engine) @handler_ids = nil end + app.syskit_log_transfer_shutdown end def default_loader diff --git a/test/roby_app/test_plugin.rb b/test/roby_app/test_plugin.rb index 85b155727..d1a73f79d 100644 --- a/test/roby_app/test_plugin.rb +++ b/test/roby_app/test_plugin.rb @@ -225,7 +225,7 @@ def perform_app_assertion(result) after do Syskit.conf.log_rotation_period = nil Syskit.conf.log_transfer.ip = nil - app.syskit_log_transfer_cleanup + app.syskit_log_transfer_shutdown end it "rotates logs and returns which logs were rotated" do @@ -256,7 +256,7 @@ def rotate_log end it "returns the list of process servers whose logs we want to transfer" do - app.syskit_log_transfer_setup + app.syskit_log_transfer_prepare conf = Syskit.conf.process_server_config_for("localhost") flexmock(conf).should_receive(supports_log_transfer?: true) @@ -266,7 +266,7 @@ def rotate_log it "ignores local process servers if they have the same directory than "\ "the transfer's target dir" do Syskit.conf.log_transfer.target_dir = app.log_dir - app.syskit_log_transfer_setup + app.syskit_log_transfer_prepare conf = Syskit.conf.process_server_config_for("localhost") flexmock(conf).should_receive(supports_log_transfer?: true) @@ -288,7 +288,7 @@ def rotate_log Configuration::ProcessServerConfig.new => ["some_file"] } ) - app.syskit_log_transfer_setup + app.syskit_log_transfer_prepare flexmock(conf.client) .should_receive(:log_upload_file).explicitly .with("127.0.0.1", 42, "cert", "user", "pass", From a70e0ac51c87a3131abdcaff535153295eceff7c Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 18 Sep 2023 21:10:12 -0300 Subject: [PATCH 098/260] fix: dispose of the log rotation poll handler in shutdown --- lib/syskit/roby_app/plugin.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index fedcb4578..28af006c4 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -261,10 +261,11 @@ def self.prepare(app) @handler_ids = plug_engine_in_roby(app.execution_engine) if Syskit.conf.log_rotation_period - app.execution_engine.every(Syskit.conf.log_rotation_period) do - app.syskit_log_perform_rotation_and_transfer - app.syskit_log_transfer_poll_state - end + @log_rotation_poll_handler = + app.execution_engine.every(Syskit.conf.log_rotation_period) do + app.syskit_log_perform_rotation_and_transfer + app.syskit_log_transfer_poll_state + end end end @@ -280,6 +281,8 @@ def self.shutdown(app) unplug_engine_from_roby(@handler_ids.values, app.execution_engine) @handler_ids = nil end + + @log_rotation_poll_handler&.dispose app.syskit_log_transfer_shutdown end From c1fdf2f40c052f1b49cd8eff6cdccf1370171bb1 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Sat, 25 Nov 2023 15:05:13 -0300 Subject: [PATCH 099/260] fix: instanciate data accessors on copy We found out a bug where data accesors wouldnt be reinstanciated on a copy, causing find_registered_data_reader/writer to point to data accessors from the old task instance. --- lib/syskit/component.rb | 17 +++++++++++------ test/test_component.rb | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/lib/syskit/component.rb b/lib/syskit/component.rb index d19816bb9..b0d18e29b 100644 --- a/lib/syskit/component.rb +++ b/lib/syskit/component.rb @@ -53,12 +53,7 @@ def initialize(**arguments) super @requirements = InstanceRequirements.new - @registered_data_writers = - instanciate_data_accessors(model.each_data_writer) - @registered_data_readers = - instanciate_data_accessors(model.each_data_reader) - @data_readers = @registered_data_readers.values - @data_writers = @registered_data_writers.values + setup_data_accessors end def initialize_copy(source) @@ -66,6 +61,16 @@ def initialize_copy(source) @requirements = @requirements.dup specialize if source.specialized_model? duplicate_missing_services_from(source) + setup_data_accessors + end + + def setup_data_accessors + @registered_data_writers = + instanciate_data_accessors(model.each_data_writer) + @registered_data_readers = + instanciate_data_accessors(model.each_data_reader) + @data_readers = @registered_data_readers.values + @data_writers = @registered_data_writers.values end def create_fresh_copy diff --git a/test/test_component.rb b/test/test_component.rb index dc3e12948..136c7ff2f 100644 --- a/test/test_component.rb +++ b/test/test_component.rb @@ -735,6 +735,13 @@ @task = syskit_stub_deploy_and_configure(@task_m) end + it "instanciates new data readers on dup" do + @task_m.data_reader @task_m.out_port, as: "test" + task1 = @task_m.new + task2 = task1.dup + refute_equal task1.test_reader, task2.test_reader + end + describe "without a given accessor name" do it "creates an unbound accessor" do reader = @task.data_reader(@task_m.out_port) @@ -960,6 +967,13 @@ @task = syskit_stub_deploy_and_configure(@task_m) end + it "instanciates new data writers on dup" do + @task_m.data_writer @task_m.in_port, as: "test" + task1 = @task_m.new + task2 = task1.dup + refute_equal task1.test_writer, task2.test_writer + end + describe "without a given accessor name" do it "creates an unbound accessor" do writer = @task.data_writer(@task_m.in_port) From 449195b086026fb8e389310d60c3905fdf3c5a81 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 20 Nov 2023 14:28:26 -0300 Subject: [PATCH 100/260] fix: remove all the "logger X does not exist" warnings that appear in unit tests --- lib/syskit/deployment.rb | 11 ++++++ lib/syskit/network_generation/logger.rb | 1 + lib/syskit/roby_app/configuration.rb | 46 ++++++++++++++-------- lib/syskit/roby_app/plugin.rb | 9 +++-- lib/syskit/test/network_manipulation.rb | 6 ++- lib/syskit/test/self.rb | 4 +- test/network_generation/test_logger.rb | 6 ++- test/roby_app/test_log_transfer_manager.rb | 4 +- 8 files changed, 61 insertions(+), 26 deletions(-) diff --git a/lib/syskit/deployment.rb b/lib/syskit/deployment.rb index 2da1648f9..6e06258ca 100644 --- a/lib/syskit/deployment.rb +++ b/lib/syskit/deployment.rb @@ -33,6 +33,7 @@ class Deployment < ::Roby::Task # rubocop:disable Metrics/ClassLength argument :ready_polling_period, default: 0.1 argument :logger_task, default: nil argument :logger_name, default: nil + argument :logging_enabled, default: nil argument :read_only, default: nil # The underlying process object @@ -369,6 +370,8 @@ def logger_name # @return [TaskContext,nil] either the logging task, or nil if this # deployment has none def logger_task + return unless logging_enabled? + if arguments[:logger_task] @logger_task = arguments[:logger_task] elsif @logger_task&.reusable? @@ -929,5 +932,13 @@ def opportunistic_recovery_from_quarantine # cleaned up, but stopped). Kill the process to avoid further damage kill! if running? end + + def logging_enabled? + if logging_enabled.nil? + process_server_config.logging_enabled? + else + logging_enabled + end + end end end diff --git a/lib/syskit/network_generation/logger.rb b/lib/syskit/network_generation/logger.rb index ff0ef2092..aa64c30c7 100644 --- a/lib/syskit/network_generation/logger.rb +++ b/lib/syskit/network_generation/logger.rb @@ -103,6 +103,7 @@ def self.add_logging_to_network(engine, work_plan) required_loggers = [] engine.deployment_tasks.each do |deployment| next unless deployment.plan + next unless deployment.logging_enabled? required_logging_ports = [] required_connections = [] diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index 567a64eb3..5d9075cb0 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -391,7 +391,7 @@ def sim_process_server_config_for(name) app.default_loader, task_context_class: Orocos::RubyTasks::StubTaskContext ) - register_process_server(sim_name, mng) + register_process_server(sim_name, mng, logging_enabled: false) end process_server_config_for(sim_name) end @@ -603,23 +603,31 @@ def connect_to_orocos_process_server( client end - ProcessServerConfig = Struct.new :name, :client, :log_dir, :host_id, :supports_log_transfer do - def on_localhost? - host_id == "localhost" || host_id == "syskit" - end + ProcessServerConfig = + Struct.new :name, :client, :log_dir, :host_id, :supports_log_transfer, + :logging_enabled, + keyword_init: true do + def on_localhost? + host_id == "localhost" || host_id == "syskit" + end - def in_process? - host_id == "syskit" - end + def in_process? + host_id == "syskit" + end - def loader - client.loader - end + def loader + client.loader + end + + def supports_log_transfer? + supports_log_transfer + end + + def logging_enabled? + logging_enabled + end - def supports_log_transfer? - supports_log_transfer end - end # Make a process server available to syskit # @@ -628,12 +636,18 @@ def supports_log_transfer? # to conform to the API of {Orocos::Remotes::Client} # @param [String] log_dir the path to the server's log directory # @return [ProcessServerConfig] - def register_process_server(name, client, log_dir = nil, host_id: name) + def register_process_server( + name, client, log_dir = nil, host_id: name, + logging_enabled: true + ) if process_servers[name] raise ArgumentError, "there is already a process server registered as #{name}, call #remove_process_server first" end - ps = ProcessServerConfig.new(name, client, log_dir, host_id) + ps = ProcessServerConfig.new( + name: name, client: client, log_dir: log_dir, host_id: host_id, + logging_enabled: logging_enabled + ) process_servers[name] = ps ps end diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index 28af006c4..bd778e796 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -108,9 +108,12 @@ def self.setup(app) "ros", fake_client, app.log_dir, host_id: "syskit" ) elsif Syskit.conf.define_default_process_managers? - Syskit.conf.register_process_server("ruby_tasks", - Orocos::RubyTasks::ProcessManager.new(app.default_loader), - app.log_dir, host_id: "syskit") + Syskit.conf.register_process_server( + "ruby_tasks", + Orocos::RubyTasks::ProcessManager.new(app.default_loader), + app.log_dir, + host_id: "syskit", logging_enabled: !app.testing? + ) Syskit.conf.register_process_server( "unmanaged_tasks", UnmanagedTasksManager.new, app.log_dir diff --git a/lib/syskit/test/network_manipulation.rb b/lib/syskit/test/network_manipulation.rb index d4540000a..48b8831ae 100644 --- a/lib/syskit/test/network_manipulation.rb +++ b/lib/syskit/test/network_manipulation.rb @@ -353,14 +353,16 @@ def syskit_stub_deployment_model( # @param [Boolean] read_only (see #syskit_stub_configured_deployment) def syskit_stub_deployment( # rubocop:disable Metrics/ParameterLists name = "deployment", deployment_model = nil, - task_model: nil, read_only: [], + task_model: nil, read_only: [], logging_enabled: nil, remote_task: syskit_stub_resolves_remote_tasks?, &block ) deployment_model ||= syskit_stub_configured_deployment( task_model, name, read_only: read_only, remote_task: remote_task, &block ) - task = deployment_model.new(process_name: name, on: "stubs") + task = deployment_model.new( + process_name: name, on: "stubs", logging_enabled: logging_enabled + ) plan.add_permanent_task(task) task end diff --git a/lib/syskit/test/self.rb b/lib/syskit/test/self.rb index 0a15e619e..9dc34d18c 100644 --- a/lib/syskit/test/self.rb +++ b/lib/syskit/test/self.rb @@ -86,8 +86,8 @@ def setup "stubs", Orocos::RubyTasks::ProcessManager.new( Roby.app.default_loader, - task_context_class: Orocos::RubyTasks::StubTaskContext - ), "", host_id: "syskit" + task_context_class: Orocos::RubyTasks::StubTaskContext, + ), "", host_id: "syskit", logging_enabled: false ) Syskit.conf.logs.create_configuration_log( File.join(app.log_dir, "properties") diff --git a/test/network_generation/test_logger.rb b/test/network_generation/test_logger.rb index 0f8f6c189..9bc43e919 100644 --- a/test/network_generation/test_logger.rb +++ b/test/network_generation/test_logger.rb @@ -142,7 +142,9 @@ end it "sets up logging while sharing a logger across deployments" do - deployment2 = syskit_stub_deployment("deployment2", deployment_m) + deployment2 = syskit_stub_deployment( + "deployment2", deployment_m, logging_enabled: true + ) task2 = deployment2.task "task" syskit_engine.should_receive(:deployment_tasks) .and_return([@deployment, deployment2]) @@ -278,6 +280,6 @@ def create_deployment_model_with_logger task "task", task_m.orogen_model task "deployment_Logger", logger_m.orogen_model end - syskit_stub_deployment("deployment", deployment_m) + syskit_stub_deployment("deployment", deployment_m, logging_enabled: true) end end diff --git a/test/roby_app/test_log_transfer_manager.rb b/test/roby_app/test_log_transfer_manager.rb index 862d1358d..94b57712c 100644 --- a/test/roby_app/test_log_transfer_manager.rb +++ b/test/roby_app/test_log_transfer_manager.rb @@ -101,7 +101,9 @@ def create_process_server client = RemoteProcesses::Client.new("localhost", server.port) log_dir = config_log_dir(client) - config = Configuration::ProcessServerConfig.new("test", client, log_dir) + config = Configuration::ProcessServerConfig.new( + name: "test", client: client, log_dir: log_dir + ) @server_threads << thread @process_servers << config config From 27b25c208f4af6a3d4a3df807c66b76ade6d23a5 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 20 Nov 2023 14:29:54 -0300 Subject: [PATCH 101/260] chore: remove unused name_service argument to RemoteProcesses::Client --- lib/syskit/roby_app/remote_processes/client.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/syskit/roby_app/remote_processes/client.rb b/lib/syskit/roby_app/remote_processes/client.rb index 2ea47df52..106691022 100644 --- a/lib/syskit/roby_app/remote_processes/client.rb +++ b/lib/syskit/roby_app/remote_processes/client.rb @@ -48,9 +48,6 @@ class StartupFailed < RuntimeError; end attr_reader :server_pid # A string that allows to uniquely identify this process server attr_reader :host_id - # The name service object that allows to resolve tasks from this process - # server - attr_reader :name_service def to_s "#" @@ -62,16 +59,12 @@ def inspect # Connects to the process server at +host+:+port+ # - # @option options [Orocos::NameService] :name_service - # (Orocos.name_service). The name service object that should be used - # to resolve tasks started by this process server # @option options [OroGen::Loaders::Base] :root_loader # (Orocos.default_loader). The loader object that should be used as # root for this client's loader def initialize( host = "localhost", port = DEFAULT_PORT, response_timeout: 10, root_loader: Orocos.default_loader, - name_service: Orocos.name_service ) @host = host @port = port @@ -86,7 +79,6 @@ def initialize( socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true) socket.fcntl(Fcntl::FD_CLOEXEC, 1) - @name_service = name_service begin @server_pid = pid rescue EOFError From 8cfeae3bff9b1dd0f0c0b901a9a9fe9872126413 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 20 Nov 2023 15:40:45 -0300 Subject: [PATCH 102/260] feat: make registration on the CORBA name service optional The main target of this feature is to disable it in unit tests, thus avoiding polluting the name service unnecessarily, removing collision (re-binding) warnings if running tests in parallel --- lib/syskit/deployment.rb | 12 ++- lib/syskit/roby_app/configuration.rb | 28 +++++-- lib/syskit/roby_app/plugin.rb | 3 +- .../roby_app/remote_processes/client.rb | 4 + lib/syskit/test/self.rb | 13 ++-- .../remote_processes/test_remote_processes.rb | 77 +++++++++++++++++-- test/roby_app/test_log_transfer_manager.rb | 3 +- test/test_deployment.rb | 52 +++++++++++-- test/test_ruby_tasks_context.rb | 4 +- 9 files changed, 166 insertions(+), 30 deletions(-) diff --git a/lib/syskit/deployment.rb b/lib/syskit/deployment.rb index 6e06258ca..4f9ab92be 100644 --- a/lib/syskit/deployment.rb +++ b/lib/syskit/deployment.rb @@ -34,6 +34,7 @@ class Deployment < ::Roby::Task # rubocop:disable Metrics/ClassLength argument :logger_task, default: nil argument :logger_name, default: nil argument :logging_enabled, default: nil + argument :register_on_name_server, default: nil argument :read_only, default: nil # The underlying process object @@ -295,7 +296,8 @@ def task(name, syskit_task_model = nil) spawn_options = spawn_options.merge( output: "%m-%p.txt", wait: false, - cmdline_args: options + cmdline_args: options, + register_on_name_server: register_on_name_server ) if log_dir @@ -940,5 +942,13 @@ def logging_enabled? logging_enabled end end + + def register_on_name_server? + if register_on_name_server.nil? + process_server_config.register_on_name_server? + else + register_on_name_server + end + end end end diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index 5d9075cb0..e609c1b8a 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -391,7 +391,10 @@ def sim_process_server_config_for(name) app.default_loader, task_context_class: Orocos::RubyTasks::StubTaskContext ) - register_process_server(sim_name, mng, logging_enabled: false) + register_process_server( + sim_name, mng, + logging_enabled: false, register_on_name_server: false + ) end process_server_config_for(sim_name) end @@ -544,7 +547,7 @@ def disconnect; end def connect_to_orocos_process_server( name, host, port: Syskit::RobyApp::RemoteProcesses::DEFAULT_PORT, log_dir: nil, result_dir: nil, host_id: nil, - name_service: Orocos.name_service, + name_service: nil, model_only_server: only_load_models? || (app.simulation? && app.single?) ) if log_dir || result_dir @@ -556,6 +559,13 @@ def connect_to_orocos_process_server( ) end + if name_service + Roby.warn_deprecated( + "the name_service argument to connect_to_orocos_process_server is "\ + "unused, and will be removed in the future" + ) + end + if model_only_server client = ModelOnlyServer.new(app.default_loader) register_process_server( @@ -587,9 +597,7 @@ def connect_to_orocos_process_server( self.disables_local_process_server = (host == "localhost") client = Syskit::RobyApp::RemoteProcesses::Client.new( - host, port, - root_loader: app.default_loader, - name_service: name_service + host, port, root_loader: app.default_loader ) client.create_log_dir( log_dir, Roby.app.time_tag, @@ -605,7 +613,7 @@ def connect_to_orocos_process_server( ProcessServerConfig = Struct.new :name, :client, :log_dir, :host_id, :supports_log_transfer, - :logging_enabled, + :logging_enabled, :register_on_name_server, keyword_init: true do def on_localhost? host_id == "localhost" || host_id == "syskit" @@ -627,6 +635,9 @@ def logging_enabled? logging_enabled end + def register_on_name_server? + register_on_name_server + end end # Make a process server available to syskit @@ -638,7 +649,7 @@ def logging_enabled? # @return [ProcessServerConfig] def register_process_server( name, client, log_dir = nil, host_id: name, - logging_enabled: true + logging_enabled: true, register_on_name_server: true ) if process_servers[name] raise ArgumentError, "there is already a process server registered as #{name}, call #remove_process_server first" @@ -646,7 +657,8 @@ def register_process_server( ps = ProcessServerConfig.new( name: name, client: client, log_dir: log_dir, host_id: host_id, - logging_enabled: logging_enabled + logging_enabled: logging_enabled, + register_on_name_server: register_on_name_server ) process_servers[name] = ps ps diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index bd778e796..c167d4fd9 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -112,7 +112,8 @@ def self.setup(app) "ruby_tasks", Orocos::RubyTasks::ProcessManager.new(app.default_loader), app.log_dir, - host_id: "syskit", logging_enabled: !app.testing? + host_id: "syskit", logging_enabled: !app.testing?, + register_on_name_server: !app.testing? ) Syskit.conf.register_process_server( diff --git a/lib/syskit/roby_app/remote_processes/client.rb b/lib/syskit/roby_app/remote_processes/client.rb index 106691022..07222e451 100644 --- a/lib/syskit/roby_app/remote_processes/client.rb +++ b/lib/syskit/roby_app/remote_processes/client.rb @@ -65,6 +65,7 @@ def inspect def initialize( host = "localhost", port = DEFAULT_PORT, response_timeout: 10, root_loader: Orocos.default_loader, + register_on_name_server: true ) @host = host @port = port @@ -90,6 +91,7 @@ def initialize( @processes = {} @death_queue = [] @host_id = "#{host}:#{port}:#{server_pid}" + @register_on_name_server = register_on_name_server @response_timeout = response_timeout end @@ -177,6 +179,8 @@ def start(process_name, deployment, name_mappings = {}, options = {}) deployment_model, options.delete(:prefix) ) name_mappings = prefix_mappings.merge(name_mappings) + options[:register_on_name_server] = + options.fetch(:register_on_name_server, @register_on_name_server) socket.write(COMMAND_START) Marshal.dump( diff --git a/lib/syskit/test/self.rb b/lib/syskit/test/self.rb index 9dc34d18c..8faca5ced 100644 --- a/lib/syskit/test/self.rb +++ b/lib/syskit/test/self.rb @@ -82,12 +82,15 @@ def setup Syskit::RobyApp::Plugin.connect_to_local_process_server end + stubs_process_manager = Orocos::RubyTasks::ProcessManager.new( + Roby.app.default_loader, + task_context_class: Orocos::RubyTasks::StubTaskContext + ) + Syskit.conf.register_process_server( - "stubs", - Orocos::RubyTasks::ProcessManager.new( - Roby.app.default_loader, - task_context_class: Orocos::RubyTasks::StubTaskContext, - ), "", host_id: "syskit", logging_enabled: false + "stubs", stubs_process_manager, "", + host_id: "syskit", logging_enabled: false, + register_on_name_server: false ) Syskit.conf.logs.create_configuration_log( File.join(app.log_dir, "properties") diff --git a/test/roby_app/remote_processes/test_remote_processes.rb b/test/roby_app/remote_processes/test_remote_processes.rb index 10e8e6ec3..2acad7bbe 100644 --- a/test/roby_app/remote_processes/test_remote_processes.rb +++ b/test/roby_app/remote_processes/test_remote_processes.rb @@ -130,8 +130,46 @@ "syskit_tests_empty", "syskit_tests_empty", {}, oro_logfile: nil, output: "/dev/null" ) + assert_same deployment, process.model end + + it "registers the task on the name server if directed to do so" do + project = OroGen::Spec::Project.new(root_loader) + deployment = project.deployment "syskit_tests_empty" + root_loader.register_deployment_model(deployment) + task_name = "syskit-remote-process-tests-#{Process.pid}" + process = client.start( + "syskit_tests_empty", "syskit_tests_empty", + { "syskit_tests_empty" => task_name }, + oro_logfile: nil, output: "/tmp/out", + register_on_name_server: true + ) + result = wait_running_process(process) + task = Orocos.allow_blocking_calls { Orocos.name_service.get task_name } + assert_equal result[:iors][task_name], + task.ior + end + + it "does not register the task on the name server if registration is disabled" do + project = OroGen::Spec::Project.new(root_loader) + deployment = project.deployment "syskit_tests_empty" + root_loader.register_deployment_model(deployment) + task_name = "syskit-remote-process-tests-#{Process.pid}" + process = client.start( + "syskit_tests_empty", "syskit_tests_empty", + { "syskit_tests_empty" => task_name }, + oro_logfile: nil, output: "/tmp/out", + register_on_name_server: false + ) + result = wait_running_process(process) + task = Orocos.allow_blocking_calls { Orocos.name_service.get task_name } + refute_equal result[:iors][task_name], + task.ior + rescue Orocos::NotFound + # Expected behavior + assert(true) + end end describe "waits for the process to be running" do @@ -140,20 +178,16 @@ end it "returns a hash with information about a process and its tasks" do - client.start( + process = client.start( "syskit_tests_empty", "syskit_tests_empty", { "syskit_tests_empty" => "syskit_tests_empty" }, oro_logfile: nil, output: "/dev/null" ) - result = nil - loop do - result = client.wait_running("syskit_tests_empty") - break if result["syskit_tests_empty"]&.key?(:iors) - end + result = wait_running_process(process) assert_match( /^IOR/, - result["syskit_tests_empty"][:iors]["syskit_tests_empty"] + result[:iors]["syskit_tests_empty"] ) end @@ -472,4 +506,33 @@ def start_and_connect_to_server start_server connect_to_server end + + def wait_running_process(process, timeout: 5) + deadline = Time.now + timeout + while Time.now < deadline + if (r = query_process_running(process)) + return r + end + + sleep 0.01 + end + flunk("did not manage to get a running #{process} in #{timeout} seconds") + end + + # Query {#client} to check whether the given process is running + # + # @return [nil,Hash] nil if the process is not ready, and the process-specific hash + # returned by the process server otherwise + # @raise if the process finished or died, or if the remote process server reports + # an error + def query_process_running(process) + result = client.wait_running(process.name) + return unless (r = result[process.name]) + return r if r.key?(:iors) + + result = client.wait_termination(0) + raise "process #{process.name} unexpectedly terminated" if result[process] + + raise r[:error] + end end diff --git a/test/roby_app/test_log_transfer_manager.rb b/test/roby_app/test_log_transfer_manager.rb index 94b57712c..8add4f50f 100644 --- a/test/roby_app/test_log_transfer_manager.rb +++ b/test/roby_app/test_log_transfer_manager.rb @@ -102,7 +102,8 @@ def create_process_server client = RemoteProcesses::Client.new("localhost", server.port) log_dir = config_log_dir(client) config = Configuration::ProcessServerConfig.new( - name: "test", client: client, log_dir: log_dir + name: "test", client: client, log_dir: log_dir, + logging_enabled: false, register_on_name_server: false ) @server_threads << thread @process_servers << config diff --git a/test/test_deployment.rb b/test/test_deployment.rb index 8b57f4592..1b37adf19 100644 --- a/test/test_deployment.rb +++ b/test/test_deployment.rb @@ -719,15 +719,19 @@ def initialize_remote_handles(handles); end end describe "using the ruby process server" do - attr_reader :task_m, :deployment_m, :deployment + attr_reader :task_name, :task_m, :deployment_m, :deployment before do - Syskit.conf.register_process_server("test", Orocos::RubyTasks::ProcessManager.new, "") + Syskit.conf.register_process_server( + "test", Orocos::RubyTasks::ProcessManager.new, "", + register_on_name_server: false + ) task_m = @task_m = TaskContext.new_submodel do input_port "in", "/double" output_port "out", "/double" end + @task_name = task_name = "syskit-ruby-tasks-test-#{Process.pid}" @deployment_m = Deployment.new_submodel(name: "deployment") do - task "task", task_m.orogen_model + task task_name, task_m.orogen_model end Syskit.conf.process_server_for("test") .register_deployment_model(deployment_m.orogen_model) @@ -736,18 +740,54 @@ def initialize_remote_handles(handles); end .to { emit deployment.ready_event } end it "can start tasks defined on a ruby process server" do - task = deployment.task("task") + task = deployment.task(task_name) assert task.orocos_task assert "task", task.orocos_task.name end it "sets the orocos_task attribute to a RubyTaskContext" do - assert_kind_of Orocos::RubyTasks::TaskContext, deployment.task("task").orocos_task + assert_kind_of Orocos::RubyTasks::TaskContext, + deployment.task(task_name).orocos_task end it "makes sure that the Ruby tasks are disposed when the deployment is stopped" do - flexmock(deployment.task("task").orocos_task).should_receive(:dispose).once.pass_thru + flexmock(deployment.task(task_name).orocos_task) + .should_receive(:dispose).once.pass_thru expect_execution { deployment.stop! } .to { emit deployment.stop_event } end + it "does not register the task on the name server by default, "\ + "as specified on the process server config" do + task = Orocos.allow_blocking_calls { Orocos.name_service.get @task_name } + # Check for equality in case we have a leftover from another failed test + refute_equal( + task.ior, deployment.remote_task_handles[@task_name].handle.ior + ) + rescue Orocos::NotFound + # Expected + end + it "registers the task on the name server if told to do so" do + expect_execution { deployment.stop! }.to { emit deployment.stop_event } + + deployment = deployment_m.new(on: "test", register_on_name_server: true) + plan.add(deployment) + expect_execution { deployment.start! }.to { emit deployment.ready_event } + task = Orocos.allow_blocking_calls { Orocos.name_service.get @task_name } + assert_equal( + task.ior, + deployment.remote_task_handles[@task_name].handle.ior + ) + end + it "deregisters the task on stop" do + expect_execution { deployment.stop! }.to { emit deployment.stop_event } + + deployment = deployment_m.new(on: "test", register_on_name_server: true) + plan.add(deployment) + expect_execution { deployment.start! }.to { emit deployment.ready_event } + expect_execution { deployment.stop! }.to { emit deployment.stop_event } + Orocos.allow_blocking_calls { Orocos.name_service.get @task_name } + flunk("task name was not deregistered on stop") + rescue Orocos::NotFound + # Expected + end end stub_process_server_deployment_helpers = Module.new do diff --git a/test/test_ruby_tasks_context.rb b/test/test_ruby_tasks_context.rb index afe510f3d..3bf6bff6d 100644 --- a/test/test_ruby_tasks_context.rb +++ b/test/test_ruby_tasks_context.rb @@ -27,7 +27,9 @@ module Syskit assert_kind_of RubyTaskContext, task assert_equal "test", task.orocos_name - remote_task = Orocos.allow_blocking_calls { Orocos.name_service.get("test") } + remote_task = Orocos.allow_blocking_calls do + Orocos::TaskContext.new(task.orocos_task.ior) + end assert_equal remote_task, task.orocos_task Orocos.allow_blocking_calls do assert_equal remote_task.in, task.orocos_task.in From d043b0e50e92fbf1a051681c1a29372fb90c541c Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 4 Dec 2023 15:59:19 -0300 Subject: [PATCH 103/260] fix: default for register_on_name_server was always `nil` (falsy) in Deployment Make sure we pick the process server default. --- lib/syskit/deployment.rb | 2 +- test/test_deployment.rb | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/lib/syskit/deployment.rb b/lib/syskit/deployment.rb index 4f9ab92be..68cbd8753 100644 --- a/lib/syskit/deployment.rb +++ b/lib/syskit/deployment.rb @@ -297,7 +297,7 @@ def task(name, syskit_task_model = nil) output: "%m-%p.txt", wait: false, cmdline_args: options, - register_on_name_server: register_on_name_server + register_on_name_server: register_on_name_server? ) if log_dir diff --git a/test/test_deployment.rb b/test/test_deployment.rb index 1b37adf19..b9cc05117 100644 --- a/test/test_deployment.rb +++ b/test/test_deployment.rb @@ -381,6 +381,80 @@ def mock_raw_port(task, port_name) .join_all_waiting_work(false) .to { not_emit deployment_task.ready_event } end + + describe "name server registration" do + before do + @orocos_task = Orocos.allow_blocking_calls do + Orocos::RubyTasks::TaskContext.new "test" + end + process.tasks["mapped_task_name"] = @orocos_task + end + + after do + @orocos_task.dispose + end + + describe "while enabled on the process server config" do + before do + flexmock(@process_server_config)\ + .should_receive(register_on_name_server?: true) + end + + it "spawns the deployment with name service if "\ + "register_on_name_server is true" do + flexmock(deployment_task, register_on_name_server: true) + expect_registers_on_name_server(true) + end + it "spawns the deployment without name service if "\ + "register_on_name_server is false" do + flexmock(deployment_task, register_on_name_server: false) + expect_registers_on_name_server(false) + end + it "spawns the deployment with name service if "\ + "register_on_name_server is nil" do + flexmock(deployment_task, register_on_name_server: nil) + expect_registers_on_name_server(true) + end + end + + describe "while disabled on the process server config" do + before do + flexmock(@process_server_config)\ + .should_receive(register_on_name_server?: false) + end + + it "spawns the deployment with name service if "\ + "register_on_name_server is true" do + flexmock(deployment_task, register_on_name_server: true) + expect_registers_on_name_server(true) + end + it "spawns the deployment without name service if "\ + "register_on_name_server is false" do + flexmock(deployment_task, register_on_name_server: false) + expect_registers_on_name_server(false) + end + it "spawns the deployment without name service if "\ + "register_on_name_server is nil" do + flexmock(deployment_task, register_on_name_server: nil) + expect_registers_on_name_server(false) + end + end + + def expect_registers_on_name_server(enabled) + check = ->(h) { !(enabled ^ h[:register_on_name_server]) } + + flexmock(@process_server) + .should_receive(:start) + .once.with(any, any, any, check) + .pass_thru + assert_deployment_ready(deployment_task) + end + + def assert_deployment_ready(deployment_task) + expect_execution { deployment_task.start! } + .to { emit deployment_task.ready_event } + end + end end describe "monitoring for ready" do From 8791c3710a00c29bcde1526f81558d8c214cdfef Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 5 Dec 2023 10:28:05 -0300 Subject: [PATCH 104/260] fix: disable name server registration for stubs It was disabled in Syskit's self-test stubs process server, but I missed the one for the bundles' tests. This was hidden by the bug that actually disabled name service registration for all deployments. --- lib/syskit/test/spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/syskit/test/spec.rb b/lib/syskit/test/spec.rb index cecb2652b..ce5aa529e 100644 --- a/lib/syskit/test/spec.rb +++ b/lib/syskit/test/spec.rb @@ -19,7 +19,8 @@ def setup loader, task_context_class: Orocos::RubyTasks::StubTaskContext ) Syskit.conf.register_process_server( - "stubs", stub_manager, nil, host_id: "syskit" + "stubs", stub_manager, nil, + host_id: "syskit", register_on_name_server: false ) super From daef0c4b89c5c203ac1d6ceae5c7eeed2ef2968a Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 5 Dec 2023 10:48:11 -0300 Subject: [PATCH 105/260] feat: allow to disable usage of the CORBA naming service --- lib/syskit/cli/gen/syskit_app/config/init.rb | 3 +++ lib/syskit/roby_app/configuration.rb | 21 ++++++++++++++++++++ lib/syskit/roby_app/plugin.rb | 6 +++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/syskit/cli/gen/syskit_app/config/init.rb b/lib/syskit/cli/gen/syskit_app/config/init.rb index 33248f19b..c82375e34 100644 --- a/lib/syskit/cli/gen/syskit_app/config/init.rb +++ b/lib/syskit/cli/gen/syskit_app/config/init.rb @@ -20,6 +20,9 @@ # Set to false to disable old-style task model export (using constants) OroGen.syskit_model_constant_registration = true +# Whether Syskit's internal ruby task should be registered on the CORBA naming +# service. +Syskit.conf.register_self_on_name_server = false # Set the module's name. It is normally inferred from the app name, and the app # name is inferred from the base directory name (e.g. an app located in diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index e609c1b8a..4be2c359b 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -143,6 +143,7 @@ def initialize(app) @auto_restart_deployments_with_quarantines = false @exception_transition_timeout = 20.0 @kill_all_on_process_server_connection = false + @register_self_on_name_server = (ENV["SYSKIT_REGISTER_SELF_ON_NAME_SERVER"] != "0") @log_rotation_period = nil @log_transfer = LogTransferManager::Configuration.new( @@ -161,6 +162,26 @@ def initialize(app) self.export_types = true end + # Whether syskit's very own ruby task should be registered on the + # CORBA naming service + # + # It is true for historical reasons. Switch globally to false by + # setting the SYSKIT_REGISTER_SELF_ON_NAME_SERVER environment + # variable to 0, or per-app by using this writer + # + # @see {#register_self_on_name_server?} + attr_writer :register_self_on_name_server + + # Whether syskit's very own ruby task should be registered on the + # CORBA naming service + # + # It is true for historical reasons. Switch globally to false by + # setting the SYSKIT_REGISTER_SELF_ON_NAME_SERVER environment + # variable to 0, or per-app by using {#register_self_on_name_server=} + def register_self_on_name_server? + @register_syskit_on_name_server + end + def create_subfield(name) Roby::OpenStruct.new(model, self, name) end diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index c167d4fd9..be551f08c 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -139,7 +139,11 @@ def self.setup(app) # Change to the log dir so that the IOR file created by # the CORBA bindings ends up there Dir.chdir(app.log_dir) do - Orocos.initialize + Orocos.initialize( + register_on_name_server: + Syskit.conf.register_self_on_name_server?, + default_corba_name_server: false + ) if Orocos::ROS.enabled? Orocos::ROS.initialize Orocos::ROS.roscore_start(:wait => true) From c6403641d0f0e8209072e7af8f758e7298c7674d Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 5 Dec 2023 10:48:20 -0300 Subject: [PATCH 106/260] fix: reduce length of Plugin.setup --- lib/syskit/roby_app/plugin.rb | 62 +++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index be551f08c..50cb6f0f9 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -96,34 +96,8 @@ def self.setup(app) ) end - if Syskit.conf.define_default_process_managers? && Syskit.conf.only_load_models? - fake_client = Configuration::ModelOnlyServer.new(app.default_loader) - Syskit.conf.register_process_server( - "ruby_tasks", fake_client, app.log_dir, host_id: "syskit" - ) - Syskit.conf.register_process_server( - "unmanaged_tasks", fake_client, app.log_dir, host_id: "syskit" - ) - Syskit.conf.register_process_server( - "ros", fake_client, app.log_dir, host_id: "syskit" - ) - elsif Syskit.conf.define_default_process_managers? - Syskit.conf.register_process_server( - "ruby_tasks", - Orocos::RubyTasks::ProcessManager.new(app.default_loader), - app.log_dir, - host_id: "syskit", logging_enabled: !app.testing?, - register_on_name_server: !app.testing? - ) - - Syskit.conf.register_process_server( - "unmanaged_tasks", UnmanagedTasksManager.new, app.log_dir - ) - - Syskit.conf.register_process_server( - "ros", Orocos::ROS::ProcessManager.new(app.ros_loader), - app.log_dir - ) + if Syskit.conf.define_default_process_managers? + define_default_process_managers(app) end ENV["ORO_LOGFILE"] = @@ -171,6 +145,38 @@ def self.setup(app) Syskit::TaskContext.define_from_orogen(rtt_core_model, register: true) end + def self.define_default_process_managers(app) + if Syskit.conf.only_load_models? + fake_client = Configuration::ModelOnlyServer.new(app.default_loader) + Syskit.conf.register_process_server( + "ruby_tasks", fake_client, app.log_dir, host_id: "syskit" + ) + Syskit.conf.register_process_server( + "unmanaged_tasks", fake_client, app.log_dir, host_id: "syskit" + ) + Syskit.conf.register_process_server( + "ros", fake_client, app.log_dir, host_id: "syskit" + ) + elsif Syskit.conf.define_default_process_managers? + Syskit.conf.register_process_server( + "ruby_tasks", + Orocos::RubyTasks::ProcessManager.new(app.default_loader), + app.log_dir, + host_id: "syskit", logging_enabled: !app.testing?, + register_on_name_server: !app.testing? + ) + + Syskit.conf.register_process_server( + "unmanaged_tasks", UnmanagedTasksManager.new, app.log_dir + ) + + Syskit.conf.register_process_server( + "ros", Orocos::ROS::ProcessManager.new(app.ros_loader), + app.log_dir + ) + end + end + def syskit_log_transfer_prepare return unless Syskit.conf.log_transfer.ip From 1bed7895747a9c8e2c1a08266d90f484c6fc2b3d Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 23 Jan 2024 16:50:28 -0300 Subject: [PATCH 107/260] fix: handle a delay between the fatal error and the component returning from stop Syskit has two communication channels with components. One is the return of state transition commands (start/stop essentially) and one is the state port. When a component goes to FATAL while it's being stopped, the fatal might have been processed before stop gets to return. If a response to the transition to fatal is to kill the deployment - something we have to do on our system - it might be that the deployment is down and the component aborted when stop returns. This is (rather obviously) not a normal condition in quarantined!, and the method was raising. Handle this case, and test for it. --- lib/syskit/task_context.rb | 4 +++- test/live/test_fatal_error.rb | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/syskit/task_context.rb b/lib/syskit/task_context.rb index c6fb4a785..343b06e60 100644 --- a/lib/syskit/task_context.rb +++ b/lib/syskit/task_context.rb @@ -1143,7 +1143,9 @@ def stop_orocos_task stop_orocos_task end promise.on_error(description: "#{self}#interrupt#error") do |error| - quarantined! unless error.kind_of?(Orocos::StateTransitionFailed) + if execution_agent && !error.kind_of?(Orocos::StateTransitionFailed) + quarantined! + end end interrupt_event.achieve_asynchronously(promise, emit_on_success: false) diff --git a/test/live/test_fatal_error.rb b/test/live/test_fatal_error.rb index b1d2ec883..1ac6a79d5 100644 --- a/test/live/test_fatal_error.rb +++ b/test/live/test_fatal_error.rb @@ -169,6 +169,23 @@ class SyskitFatalErrorTests < Syskit::Test::ComponentTest trigger_fatal_error(task) end + it "handles a delay between the fatal error and the component "\ + "returning from stop" do + task_m = OroGen.orogen_syskit_tests.FatalError + .deployed_as(default_deployment_name) + task = syskit_deploy(task_m) + task.properties.stop_return_delay_after_fatal_ms = 5_000 + syskit_configure_and_start(task) + plan.unmark_permanent_task(task.execution_agent) + flexmock(task).should_receive(:quarantined!).never + expect_execution { task.stop! } + .garbage_collect(true).join_all_waiting_work(false) + .to do + emit task.fatal_error_event + emit task.execution_agent.stop_event + end + end + it "marks itself as being in FATAL on its deployment" do task_m = OroGen.orogen_syskit_tests.FatalError .deployed_as(default_deployment_name) From 582bd1db818dff60c7b085ed521e9119aebe51e0 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 18 Dec 2023 14:26:11 -0300 Subject: [PATCH 108/260] feat: add free space management to the archiver --- lib/syskit/cli/log_runtime_archive.rb | 34 ++++++++++++++++++++++- lib/syskit/scripts/log_runtime_archive.rb | 9 ++++++ test/cli/test_log_runtime_archive.rb | 4 +++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index a4d058589..c3c4d90e7 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "archive/tar/minitar" +require "sys/filesystem" module Syskit module CLI @@ -12,17 +13,23 @@ module CLI # It depends on the syskit instance using log rotation class LogRuntimeArchive DEFAULT_MAX_ARCHIVE_SIZE = 10_000_000_000 # 10G + FREE_SPACE_LOW_LIMIT = 1_000_000_000 # 1gb + FREE_SPACE_FREED_LIMIT = 10_000_000_000 # 10gb def initialize( root_dir, target_dir, logger: LogRuntimeArchive.null_logger, - max_archive_size: DEFAULT_MAX_ARCHIVE_SIZE + max_archive_size: DEFAULT_MAX_ARCHIVE_SIZE, + free_space_low_limit: FREE_SPACE_LOW_LIMIT, + free_space_delete_until: FREE_SPACE_FREED_LIMIT ) @last_archive_index = {} @logger = logger @root_dir = root_dir @target_dir = target_dir @max_archive_size = max_archive_size + @free_space_low_limit = free_space_low_limit + @free_space_delete_until = free_space_delete_until end # Iterate over all datasets in a Roby log root folder and archive them @@ -41,6 +48,30 @@ def process_root_folder end end + # Manages folder free space + # + # The method will check if there is enough space to save more log files + # according to pre-established threshold. + # + # @param [integer] free_space_low_limit: required free space threshold, at + # which the archiver starts deleting the oldest log files + # @param [integer] free_space_delete_until: post-deletion free space, at which + # the archiver stops deleting the oldest log fil + def ensure_free_space(free_space_low_limit: @free_space_low_limit, + free_space_delete_until: @free_space_delete_until) + stat = Sys::Filesystem.stat(@target_dir) + free_space = stat.bytes_free + + return if free_space > free_space_low_limit + + candidates = each_file_from_path(path).to_a.sort + loop do |index| + stat.delete(candidates[index]) + free_space = stat.bytes_free + break if free_space >= free_space_delete_until + end + end + def process_dataset(child, full:) use_existing = true loop do @@ -64,6 +95,7 @@ def process_dataset(child, full:) end end + # Create or open an archive # # The method will find an archive to open or create, do it and diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index 474f6b281..ff33285b5 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -18,12 +18,21 @@ def self.exit_on_failure? type: :numeric, default: 600, desc: "polling period in seconds" option :max_size, type: :numeric, default: 10_000, desc: "max log size in MB" + option :free_space_low_limit, + type: :numeric, default: 1_000, desc: "start deleting files if free space is \ + below this threshold" + option :free_space_delete_until, + type: :numeric, default: 10_000, desc: "stop deleting files if free space is \ + above this threshold" default_task def watch(root_dir, target_dir) root_dir = validate_directory_exists(root_dir) target_dir = validate_directory_exists(target_dir) archiver = make_archiver(root_dir, target_dir) loop do archiver.process_root_folder + archiver.ensure_free_space( + options[:free_space_low_limit], options[:free_space_delete_until] + ) puts "Archived pending logs, sleeping #{options[:period]}s" sleep options[:period] diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 239c2dd67..45873381e 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -510,6 +510,10 @@ def should_archive_dataset(dataset, archive_basename, full:) end end + describe ".ensure_free_space" do + + end + def make_valid_folder(name) path = (@root / name) path.mkpath From b8a3bd7c1107754747a8c694bd1b78b671f72bf0 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 19 Dec 2023 11:05:06 -0300 Subject: [PATCH 109/260] feat: implement free space management tests --- lib/syskit/cli/log_runtime_archive.rb | 4 ++ manifest.xml | 1 + test/cli/test_log_runtime_archive.rb | 56 ++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index c3c4d90e7..803433c2b 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -72,6 +72,10 @@ def ensure_free_space(free_space_low_limit: @free_space_low_limit, end end + def self.size_of_file(path) + path.stat.size + end + def process_dataset(child, full:) use_existing = true loop do diff --git a/manifest.xml b/manifest.xml index 63d56ba9f..fb3f99957 100644 --- a/manifest.xml +++ b/manifest.xml @@ -16,6 +16,7 @@ + diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 45873381e..70a8fc051 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -510,8 +510,62 @@ def should_archive_dataset(dataset, archive_basename, full:) end end - describe ".ensure_free_space" do + describe "#ensure_free_space" do + before do + @target_dir = make_tmppath + 10.times { |i| (@target_dir / i.to_s).write(i.to_s) } + + @archiver = LogRuntimeArchive.new( + @root_dir, @target_dir, + free_space_low_limit: 1, + free_space_freed_limit: 10 + ) + end + + it "does nothing if there is enough free space" do + mock_target_dir_free_space(2) + assert_deleted_files([]) + end + + it "removes enough files to reach the freed limit" do + mock_target_dir_free_space(0.5) + mock_files_size([0.6, 2, 4, 6, 7]) + assert_deleted_files([0, 1, 2, 3]) + end + def mock_files_size(sizes) + sizes.each_with_index do |size, i| + flexmock(@archiver) + .should_receive(:size_of_file) + .with(@target_dir / i.to_s) + .and_return(size) + end + end + + def mock_target_dir_free_space(space) + flexmock(Sys::Filesystem) + .should_receive(:stat).with(@target_dir) + .and_return(flexmock(bytes_free: space)) + end + + def assert_deleted_files(deleted_files) + (0...10).each do |i| + if deleted_files.include?(i) + refute (@target_dir / i.to_s).exist?, + "#{i} was expected to be deleted, but has not been" + else + assert (@target_dir / i.to_s).exist?, + "#{i} was expected to be present, but got deleted" + end + end + end + end + + describe ".size_of_file" do + it "returns the size in bytes of a file" do + path = (@root / "testfile").write(" " * 10) + assert_equal 10, LogRuntimeArchive.size_of_file(path) + end end def make_valid_folder(name) From ca88afbf0888a4cebb28bb9b339edcae2888724c Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Thu, 21 Dec 2023 16:06:51 -0300 Subject: [PATCH 110/260] fix: fix functions implementation to delete first array element unit until free space is above the threshold --- lib/syskit/cli/log_runtime_archive.rb | 13 +++++++------ test/cli/test_log_runtime_archive.rb | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 803433c2b..6f5c1b7a2 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -59,21 +59,23 @@ def process_root_folder # the archiver stops deleting the oldest log fil def ensure_free_space(free_space_low_limit: @free_space_low_limit, free_space_delete_until: @free_space_delete_until) + return if free_space_low_limit >= free_space_delete_until + stat = Sys::Filesystem.stat(@target_dir) free_space = stat.bytes_free return if free_space > free_space_low_limit - candidates = each_file_from_path(path).to_a.sort - loop do |index| - stat.delete(candidates[index]) + candidates = Dir.entries(@target_dir).sort + until free_space >= free_space_delete_until + candidates.shift free_space = stat.bytes_free - break if free_space >= free_space_delete_until end end + # Returns the file size in bytes def self.size_of_file(path) - path.stat.size + File.stat(path).size end def process_dataset(child, full:) @@ -99,7 +101,6 @@ def process_dataset(child, full:) end end - # Create or open an archive # # The method will find an archive to open or create, do it and diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 70a8fc051..5e18146fe 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -518,7 +518,7 @@ def should_archive_dataset(dataset, archive_basename, full:) @archiver = LogRuntimeArchive.new( @root_dir, @target_dir, free_space_low_limit: 1, - free_space_freed_limit: 10 + free_space_delete_until: 10 ) end From 571286f948fb5fd95bb1d8be8626fc8e0c9faa6a Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Thu, 21 Dec 2023 16:31:53 -0300 Subject: [PATCH 111/260] fix: fix unit tests --- lib/syskit/cli/log_runtime_archive.rb | 4 ++-- test/cli/test_log_runtime_archive.rb | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 6f5c1b7a2..37e3b7be1 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -13,8 +13,8 @@ module CLI # It depends on the syskit instance using log rotation class LogRuntimeArchive DEFAULT_MAX_ARCHIVE_SIZE = 10_000_000_000 # 10G - FREE_SPACE_LOW_LIMIT = 1_000_000_000 # 1gb - FREE_SPACE_FREED_LIMIT = 10_000_000_000 # 10gb + FREE_SPACE_LOW_LIMIT = 1_000_000_000 # 1 G + FREE_SPACE_FREED_LIMIT = 10_000_000_000 # 10 G def initialize( root_dir, target_dir, diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 5e18146fe..e0546a24e 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -535,7 +535,7 @@ def should_archive_dataset(dataset, archive_basename, full:) def mock_files_size(sizes) sizes.each_with_index do |size, i| - flexmock(@archiver) + flexmock(LogRuntimeArchive) .should_receive(:size_of_file) .with(@target_dir / i.to_s) .and_return(size) @@ -563,7 +563,8 @@ def assert_deleted_files(deleted_files) describe ".size_of_file" do it "returns the size in bytes of a file" do - path = (@root / "testfile").write(" " * 10) + path = (@root / "testfile") + path.write(" " * 10) assert_equal 10, LogRuntimeArchive.size_of_file(path) end end From 45c096e2f148042debff8236b8c7a2c91f7d606a Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Tue, 2 Jan 2024 15:47:31 -0300 Subject: [PATCH 112/260] fix: fix file deletion --- lib/syskit/cli/log_runtime_archive.rb | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 37e3b7be1..d3052890b 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -48,7 +48,7 @@ def process_root_folder end end - # Manages folder free space + # Manages folder available space # # The method will check if there is enough space to save more log files # according to pre-established threshold. @@ -56,20 +56,23 @@ def process_root_folder # @param [integer] free_space_low_limit: required free space threshold, at # which the archiver starts deleting the oldest log files # @param [integer] free_space_delete_until: post-deletion free space, at which - # the archiver stops deleting the oldest log fil + # the archiver stops deleting the oldest log files def ensure_free_space(free_space_low_limit: @free_space_low_limit, free_space_delete_until: @free_space_delete_until) return if free_space_low_limit >= free_space_delete_until stat = Sys::Filesystem.stat(@target_dir) - free_space = stat.bytes_free + available_space = stat.bytes_free - return if free_space > free_space_low_limit + return if available_space > free_space_delete_until - candidates = Dir.entries(@target_dir).sort - until free_space >= free_space_delete_until - candidates.shift - free_space = stat.bytes_free + until available_space >= free_space_delete_until + files = @target_dir.each_child.select { |file| file.file? } + break if files.empty? + + files.sort.first.unlink + stat = Sys::Filesystem.stat(@target_dir) + available_space = stat.bytes_free end end From cae4c5a6db007ef7b90a343f416a381bf50ca9b9 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Thu, 4 Jan 2024 16:56:33 -0300 Subject: [PATCH 113/260] fix: fix unit tests --- test/cli/test_log_runtime_archive.rb | 77 ++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index e0546a24e..43741dfdb 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -512,50 +512,87 @@ def should_archive_dataset(dataset, archive_basename, full:) describe "#ensure_free_space" do before do - @target_dir = make_tmppath - 10.times { |i| (@target_dir / i.to_s).write(i.to_s) } + @archive_dir = make_tmppath + @free_space_low_limit = 1 + @free_space_delete_until = 10 + @deleted_files = [] + @mocked_files_sizes = [] + + 10.times { |i| (@archive_dir / i.to_s).write(i.to_s) } @archiver = LogRuntimeArchive.new( - @root_dir, @target_dir, - free_space_low_limit: 1, - free_space_delete_until: 10 + @root, @archive_dir, + free_space_low_limit: @free_space_low_limit, + free_space_delete_until: @free_space_delete_until ) end it "does nothing if there is enough free space" do - mock_target_dir_free_space(2) + mock_available_space(2) + @archiver.ensure_free_space( + free_space_low_limit: @free_space_low_limit, + free_space_delete_until: @free_space_delete_until + ) assert_deleted_files([]) end it "removes enough files to reach the freed limit" do - mock_target_dir_free_space(0.5) - mock_files_size([0.6, 2, 4, 6, 7]) - assert_deleted_files([0, 1, 2, 3]) + size_files = [6, 2, 4, 6, 7, 10, 3, 5, 8, 9] + mock_files_size(size_files) + mock_available_space(size_files.sum + 0.5) + + @archiver.ensure_free_space( + free_space_low_limit: @free_space_low_limit, + free_space_delete_until: @free_space_delete_until + ) + assert_deleted_files(@deleted_files) end def mock_files_size(sizes) + @mocked_files_sizes = sizes sizes.each_with_index do |size, i| flexmock(LogRuntimeArchive) .should_receive(:size_of_file) - .with(@target_dir / i.to_s) + .with(@archive_dir / i.to_s) .and_return(size) end end - def mock_target_dir_free_space(space) + def mock_available_space(total_disk_size) flexmock(Sys::Filesystem) - .should_receive(:stat).with(@target_dir) - .and_return(flexmock(bytes_free: space)) + .should_receive(:stat).with(@archive_dir) + .and_return do + flexmock( + bytes_free: compute_mocked_disk_size(total_disk_size) + ) + end + end + + def compute_mocked_disk_size(total_disk_size) + return total_disk_size if @mocked_files_sizes.empty? + + @mocked_files_sizes.each_index do |i| + if !(@archive_dir / i.to_s).exist? + total_disk_size -= @mocked_files_sizes[i] + @deleted_files.append(i) + end + end + + total_disk_size end def assert_deleted_files(deleted_files) - (0...10).each do |i| - if deleted_files.include?(i) - refute (@target_dir / i.to_s).exist?, - "#{i} was expected to be deleted, but has not been" - else - assert (@target_dir / i.to_s).exist?, - "#{i} was expected to be present, but got deleted" + if deleted_files.empty? + assert_equal 10, @archive_dir.each_child.size + else + (0..9).each do |i| + if deleted_files.include?(i) + refute (@archive_dir / i.to_s).exist?, + "#{i} was expected to be deleted, but has not been" + else + assert (@archive_dir / i.to_s).exist?, + "#{i} was expected to be present, but got deleted" + end end end end From 7bd97db1a85eb749661964da922d649563d2f1fa Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 8 Jan 2024 17:31:28 -0300 Subject: [PATCH 114/260] feat: add unit test 'break condition' test: stops removing files when there is no file in folder even if freed limit is not achieved --- test/cli/test_log_runtime_archive.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 43741dfdb..4cea6552b 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -548,6 +548,19 @@ def should_archive_dataset(dataset, archive_basename, full:) assert_deleted_files(@deleted_files) end + it "stops removing files when there is no file in folder even if freed + limit is not achieved" do + size_files = Array.new(10, 1) + mock_files_size(size_files) + mock_available_space(0.5) + + @archiver.ensure_free_space( + free_space_low_limit: 1, + free_space_delete_until: 15 + ) + assert_deleted_files([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + end + def mock_files_size(sizes) @mocked_files_sizes = sizes sizes.each_with_index do |size, i| From eb1f8d352c06b620edee09bf7d78b941ee5437dc Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 8 Jan 2024 17:32:42 -0300 Subject: [PATCH 115/260] fix: fix mocked files sizes and total available folder space --- test/cli/test_log_runtime_archive.rb | 32 +++++++--------------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 4cea6552b..515dabfde 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -515,7 +515,6 @@ def should_archive_dataset(dataset, archive_basename, full:) @archive_dir = make_tmppath @free_space_low_limit = 1 @free_space_delete_until = 10 - @deleted_files = [] @mocked_files_sizes = [] 10.times { |i| (@archive_dir / i.to_s).write(i.to_s) } @@ -537,15 +536,15 @@ def should_archive_dataset(dataset, archive_basename, full:) end it "removes enough files to reach the freed limit" do - size_files = [6, 2, 4, 6, 7, 10, 3, 5, 8, 9] + size_files = [6, 2, 1, 6, 7, 10, 3, 5, 8, 9] mock_files_size(size_files) - mock_available_space(size_files.sum + 0.5) + mock_available_space(0.5) @archiver.ensure_free_space( free_space_low_limit: @free_space_low_limit, free_space_delete_until: @free_space_delete_until ) - assert_deleted_files(@deleted_files) + assert_deleted_files([0, 1, 2, 3]) end it "stops removing files when there is no file in folder even if freed @@ -563,11 +562,8 @@ def should_archive_dataset(dataset, archive_basename, full:) def mock_files_size(sizes) @mocked_files_sizes = sizes - sizes.each_with_index do |size, i| - flexmock(LogRuntimeArchive) - .should_receive(:size_of_file) - .with(@archive_dir / i.to_s) - .and_return(size) + @mocked_files_sizes.each_with_index do |size, i| + (@archive_dir / i.to_s).write(" " * size) end end @@ -576,27 +572,15 @@ def mock_available_space(total_disk_size) .should_receive(:stat).with(@archive_dir) .and_return do flexmock( - bytes_free: compute_mocked_disk_size(total_disk_size) + bytes_free: total_disk_size ) end end - def compute_mocked_disk_size(total_disk_size) - return total_disk_size if @mocked_files_sizes.empty? - - @mocked_files_sizes.each_index do |i| - if !(@archive_dir / i.to_s).exist? - total_disk_size -= @mocked_files_sizes[i] - @deleted_files.append(i) - end - end - - total_disk_size - end - def assert_deleted_files(deleted_files) if deleted_files.empty? - assert_equal 10, @archive_dir.each_child.size + files = @archive_dir.each_child.select(&:file?) + assert_equal 10, files.size else (0..9).each do |i| if deleted_files.include?(i) From a91a3d884576145819bc82e3739e1ab7eb285cb3 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 8 Jan 2024 17:33:12 -0300 Subject: [PATCH 116/260] fix: fix rubocop --- lib/syskit/cli/log_runtime_archive.rb | 2 +- lib/syskit/scripts/log_runtime_archive.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index d3052890b..d0378c22d 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -58,7 +58,7 @@ def process_root_folder # @param [integer] free_space_delete_until: post-deletion free space, at which # the archiver stops deleting the oldest log files def ensure_free_space(free_space_low_limit: @free_space_low_limit, - free_space_delete_until: @free_space_delete_until) + free_space_delete_until: @free_space_delete_until) return if free_space_low_limit >= free_space_delete_until stat = Sys::Filesystem.stat(@target_dir) diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index ff33285b5..e2e7b5916 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -19,10 +19,10 @@ def self.exit_on_failure? option :max_size, type: :numeric, default: 10_000, desc: "max log size in MB" option :free_space_low_limit, - type: :numeric, default: 1_000, desc: "start deleting files if free space is \ + type: :numeric, default: 1_000, desc: "start deleting files if free space is \ below this threshold" option :free_space_delete_until, - type: :numeric, default: 10_000, desc: "stop deleting files if free space is \ + type: :numeric, default: 10_000, desc: "stop deleting files if free space is \ above this threshold" default_task def watch(root_dir, target_dir) root_dir = validate_directory_exists(root_dir) From 07f71be4bc521b6239d02cf83aa6f8a623d0042f Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 8 Jan 2024 17:34:41 -0300 Subject: [PATCH 117/260] fix: fix available_space loop check --- lib/syskit/cli/log_runtime_archive.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index d0378c22d..78c08c679 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -64,15 +64,14 @@ def ensure_free_space(free_space_low_limit: @free_space_low_limit, stat = Sys::Filesystem.stat(@target_dir) available_space = stat.bytes_free - return if available_space > free_space_delete_until + return if available_space > free_space_low_limit until available_space >= free_space_delete_until - files = @target_dir.each_child.select { |file| file.file? } + files = @target_dir.each_child.select(&:file?) break if files.empty? + available_space += files.sort.first.size files.sort.first.unlink - stat = Sys::Filesystem.stat(@target_dir) - available_space = stat.bytes_free end end From 324c3ce60214141b9a0732822d351f19118df834 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 8 Jan 2024 17:45:04 -0300 Subject: [PATCH 118/260] feat: add available space limits validation --- lib/syskit/cli/log_runtime_archive.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 78c08c679..77cbeeae3 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -23,6 +23,12 @@ def initialize( free_space_low_limit: FREE_SPACE_LOW_LIMIT, free_space_delete_until: FREE_SPACE_FREED_LIMIT ) + if free_space_low_limit > free_space_delete_until + raise ArgumentError, + "cannot erase files: freed limit is smaller than " \ + "low limit space." + end + @last_archive_index = {} @logger = logger @root_dir = root_dir @@ -59,8 +65,6 @@ def process_root_folder # the archiver stops deleting the oldest log files def ensure_free_space(free_space_low_limit: @free_space_low_limit, free_space_delete_until: @free_space_delete_until) - return if free_space_low_limit >= free_space_delete_until - stat = Sys::Filesystem.stat(@target_dir) available_space = stat.bytes_free From ce115f97fe8f213e48a030320a0203b96737cbaf Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Thu, 11 Jan 2024 20:18:42 -0300 Subject: [PATCH 119/260] chore: change default values --- lib/syskit/cli/log_runtime_archive.rb | 4 ++-- lib/syskit/scripts/log_runtime_archive.rb | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 77cbeeae3..7e55d7c6c 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -13,8 +13,8 @@ module CLI # It depends on the syskit instance using log rotation class LogRuntimeArchive DEFAULT_MAX_ARCHIVE_SIZE = 10_000_000_000 # 10G - FREE_SPACE_LOW_LIMIT = 1_000_000_000 # 1 G - FREE_SPACE_FREED_LIMIT = 10_000_000_000 # 10 G + FREE_SPACE_LOW_LIMIT = 5_000_000_000 # 5 G + FREE_SPACE_FREED_LIMIT = 25_000_000_000 # 25 G def initialize( root_dir, target_dir, diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index e2e7b5916..abb2318b9 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -18,11 +18,11 @@ def self.exit_on_failure? type: :numeric, default: 600, desc: "polling period in seconds" option :max_size, type: :numeric, default: 10_000, desc: "max log size in MB" - option :free_space_low_limit, - type: :numeric, default: 1_000, desc: "start deleting files if free space is \ + option :FREE_SPACE_LOW_LIMIT, + type: :numeric, default: 5_000, desc: "start deleting files if free space is \ below this threshold" - option :free_space_delete_until, - type: :numeric, default: 10_000, desc: "stop deleting files if free space is \ + option :FREE_SPACE_FREED_LIMIT, + type: :numeric, default: 25_000, desc: "stop deleting files if free space is \ above this threshold" default_task def watch(root_dir, target_dir) root_dir = validate_directory_exists(root_dir) From 60f1cf7311cb44d60241d1f71d4f2a84763508b7 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Thu, 11 Jan 2024 20:19:54 -0300 Subject: [PATCH 120/260] fix: remove unused function --- lib/syskit/cli/log_runtime_archive.rb | 5 ----- test/cli/test_log_runtime_archive.rb | 8 -------- 2 files changed, 13 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 7e55d7c6c..a559ffabf 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -79,11 +79,6 @@ def ensure_free_space(free_space_low_limit: @free_space_low_limit, end end - # Returns the file size in bytes - def self.size_of_file(path) - File.stat(path).size - end - def process_dataset(child, full:) use_existing = true loop do diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 515dabfde..603083c87 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -595,14 +595,6 @@ def assert_deleted_files(deleted_files) end end - describe ".size_of_file" do - it "returns the size in bytes of a file" do - path = (@root / "testfile") - path.write(" " * 10) - assert_equal 10, LogRuntimeArchive.size_of_file(path) - end - end - def make_valid_folder(name) path = (@root / name) path.mkpath From c8dc2bea1d222e029d9e4f2e0c1800d342cb5b87 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Thu, 11 Jan 2024 20:22:47 -0300 Subject: [PATCH 121/260] fix + chore: use default constants and ensure that archiver has enough free space --- lib/syskit/cli/log_runtime_archive.rb | 24 ++++++++++++----------- lib/syskit/scripts/log_runtime_archive.rb | 4 ++-- test/cli/test_log_runtime_archive.rb | 23 ++++------------------ 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index a559ffabf..4e537790e 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -19,11 +19,9 @@ class LogRuntimeArchive def initialize( root_dir, target_dir, logger: LogRuntimeArchive.null_logger, - max_archive_size: DEFAULT_MAX_ARCHIVE_SIZE, - free_space_low_limit: FREE_SPACE_LOW_LIMIT, - free_space_delete_until: FREE_SPACE_FREED_LIMIT + max_archive_size: DEFAULT_MAX_ARCHIVE_SIZE ) - if free_space_low_limit > free_space_delete_until + if FREE_SPACE_LOW_LIMIT > FREE_SPACE_FREED_LIMIT raise ArgumentError, "cannot erase files: freed limit is smaller than " \ "low limit space." @@ -34,8 +32,6 @@ def initialize( @root_dir = root_dir @target_dir = target_dir @max_archive_size = max_archive_size - @free_space_low_limit = free_space_low_limit - @free_space_delete_until = free_space_delete_until end # Iterate over all datasets in a Roby log root folder and archive them @@ -63,8 +59,8 @@ def process_root_folder # which the archiver starts deleting the oldest log files # @param [integer] free_space_delete_until: post-deletion free space, at which # the archiver stops deleting the oldest log files - def ensure_free_space(free_space_low_limit: @free_space_low_limit, - free_space_delete_until: @free_space_delete_until) + def ensure_free_space(free_space_low_limit = FREE_SPACE_LOW_LIMIT, + free_space_delete_until = FREE_SPACE_FREED_LIMIT) stat = Sys::Filesystem.stat(@target_dir) available_space = stat.bytes_free @@ -72,10 +68,16 @@ def ensure_free_space(free_space_low_limit: @free_space_low_limit, until available_space >= free_space_delete_until files = @target_dir.each_child.select(&:file?) - break if files.empty? + if files.empty? + Roby.warn "Cannot erase files: the folder is empty but the "\ + "available space is smaller than the threshold." + break + end - available_space += files.sort.first.size - files.sort.first.unlink + removed_file = files.min + size_removed_file = removed_file.size + removed_file.unlink + available_space += size_removed_file end end diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index abb2318b9..8d4fe2a7f 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -29,10 +29,10 @@ def self.exit_on_failure? target_dir = validate_directory_exists(target_dir) archiver = make_archiver(root_dir, target_dir) loop do - archiver.process_root_folder archiver.ensure_free_space( - options[:free_space_low_limit], options[:free_space_delete_until] + options[:FREE_SPACE_LOW_LIMIT], options[:FREE_SPACE_FREED_LIMIT] ) + archiver.process_root_folder puts "Archived pending logs, sleeping #{options[:period]}s" sleep options[:period] diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 603083c87..fbe9d318f 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -513,25 +513,16 @@ def should_archive_dataset(dataset, archive_basename, full:) describe "#ensure_free_space" do before do @archive_dir = make_tmppath - @free_space_low_limit = 1 - @free_space_delete_until = 10 @mocked_files_sizes = [] 10.times { |i| (@archive_dir / i.to_s).write(i.to_s) } - @archiver = LogRuntimeArchive.new( - @root, @archive_dir, - free_space_low_limit: @free_space_low_limit, - free_space_delete_until: @free_space_delete_until - ) + @archiver = LogRuntimeArchive.new(@root, @archive_dir) end it "does nothing if there is enough free space" do mock_available_space(2) - @archiver.ensure_free_space( - free_space_low_limit: @free_space_low_limit, - free_space_delete_until: @free_space_delete_until - ) + @archiver.ensure_free_space(1, 10) assert_deleted_files([]) end @@ -540,10 +531,7 @@ def should_archive_dataset(dataset, archive_basename, full:) mock_files_size(size_files) mock_available_space(0.5) - @archiver.ensure_free_space( - free_space_low_limit: @free_space_low_limit, - free_space_delete_until: @free_space_delete_until - ) + @archiver.ensure_free_space(1, 10) assert_deleted_files([0, 1, 2, 3]) end @@ -553,10 +541,7 @@ def should_archive_dataset(dataset, archive_basename, full:) mock_files_size(size_files) mock_available_space(0.5) - @archiver.ensure_free_space( - free_space_low_limit: 1, - free_space_delete_until: 15 - ) + @archiver.ensure_free_space(1, 15) assert_deleted_files([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) end From 6b5ef70bb1ef1f8b86d3d5013f8c3fb864c4b065 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Fri, 12 Jan 2024 15:24:52 -0300 Subject: [PATCH 122/260] fix: remove default constants set by user overridden in ensure_free_space method --- lib/syskit/cli/log_runtime_archive.rb | 17 +++++++---------- lib/syskit/scripts/log_runtime_archive.rb | 15 ++++++++------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 4e537790e..49f2d9e2b 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -13,20 +13,12 @@ module CLI # It depends on the syskit instance using log rotation class LogRuntimeArchive DEFAULT_MAX_ARCHIVE_SIZE = 10_000_000_000 # 10G - FREE_SPACE_LOW_LIMIT = 5_000_000_000 # 5 G - FREE_SPACE_FREED_LIMIT = 25_000_000_000 # 25 G def initialize( root_dir, target_dir, logger: LogRuntimeArchive.null_logger, max_archive_size: DEFAULT_MAX_ARCHIVE_SIZE ) - if FREE_SPACE_LOW_LIMIT > FREE_SPACE_FREED_LIMIT - raise ArgumentError, - "cannot erase files: freed limit is smaller than " \ - "low limit space." - end - @last_archive_index = {} @logger = logger @root_dir = root_dir @@ -59,8 +51,13 @@ def process_root_folder # which the archiver starts deleting the oldest log files # @param [integer] free_space_delete_until: post-deletion free space, at which # the archiver stops deleting the oldest log files - def ensure_free_space(free_space_low_limit = FREE_SPACE_LOW_LIMIT, - free_space_delete_until = FREE_SPACE_FREED_LIMIT) + def ensure_free_space(free_space_low_limit, free_space_delete_until) + if free_space_low_limit > free_space_delete_until + raise ArgumentError, + "cannot erase files: freed limit is smaller than " \ + "low limit space." + end + stat = Sys::Filesystem.stat(@target_dir) available_space = stat.bytes_free diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index 8d4fe2a7f..7a033f137 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -18,19 +18,20 @@ def self.exit_on_failure? type: :numeric, default: 600, desc: "polling period in seconds" option :max_size, type: :numeric, default: 10_000, desc: "max log size in MB" - option :FREE_SPACE_LOW_LIMIT, - type: :numeric, default: 5_000, desc: "start deleting files if free space is \ - below this threshold" - option :FREE_SPACE_FREED_LIMIT, - type: :numeric, default: 25_000, desc: "stop deleting files if free space is \ - above this threshold" + option :free_space_low_limit, + type: :numeric, default: 5_000, desc: "start deleting files if available \ + space is below this threshold (threshold in MB)" + option :free_space_freed_limit, + type: :numeric, default: 25_000, desc: "stop deleting files if available \ + space is above this threshold (threshold in MB)" default_task def watch(root_dir, target_dir) root_dir = validate_directory_exists(root_dir) target_dir = validate_directory_exists(target_dir) archiver = make_archiver(root_dir, target_dir) loop do archiver.ensure_free_space( - options[:FREE_SPACE_LOW_LIMIT], options[:FREE_SPACE_FREED_LIMIT] + options[:free_space_low_limit] * 1000, + options[:free_space_freed_limit] * 1000 ) archiver.process_root_folder From 42b21c991a0933d0bf31e60a9474c9d918bbb72d Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Tue, 23 Jan 2024 14:41:47 -0300 Subject: [PATCH 123/260] test: test cli deleting behavior --- lib/syskit/cli/log_runtime_archive_main.rb | 64 ++++++++++++++++++++ lib/syskit/scripts/log_runtime_archive.rb | 2 +- test/cli/test_log_runtime_archive_main.rb | 70 ++++++++++++++++++++++ 3 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 lib/syskit/cli/log_runtime_archive_main.rb create mode 100644 test/cli/test_log_runtime_archive_main.rb diff --git a/lib/syskit/cli/log_runtime_archive_main.rb b/lib/syskit/cli/log_runtime_archive_main.rb new file mode 100644 index 000000000..c70bc92bd --- /dev/null +++ b/lib/syskit/cli/log_runtime_archive_main.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require "pathname" +require "thor" +require "syskit/cli/log_runtime_archive" + +module Syskit + module CLI + class LogRuntimeArchiveMain < Thor + def self.exit_on_failure? + true + end + + desc "watch", "watch a dataset root folder and archive the datasets" + option :period, + type: :numeric, default: 600, desc: "polling period in seconds" + option :max_size, + type: :numeric, default: 10_000, desc: "max log size in MB" + option :free_space_low_limit, + type: :numeric, default: 5_000, desc: "start deleting files if available \ + space is below this threshold (threshold in MB)" + option :free_space_freed_limit, + type: :numeric, default: 25_000, desc: "stop deleting files if available \ + space is above this threshold (threshold in MB)" + default_task def watch(root_dir, target_dir) + root_dir = validate_directory_exists(root_dir) + target_dir = validate_directory_exists(target_dir) + archiver = make_archiver(root_dir, target_dir) + loop do + archiver.ensure_free_space( + options[:free_space_low_limit] * 1000, + options[:free_space_freed_limit] * 1000 + ) + archiver.process_root_folder + + puts "Archived pending logs, sleeping #{options[:period]}s" + sleep options[:period] + end + end + + no_commands do + def validate_directory_exists(dir) + dir = Pathname.new(dir) + unless dir.directory? + raise ArgumentError, "#{dir} does not exist, or is not a directory" + end + + dir + end + + def make_archiver(root_dir, target_dir) + logger = Logger.new(STDOUT) + + Syskit::CLI::LogRuntimeArchive.new( + root_dir, target_dir, + logger: logger, max_archive_size: options[:max_size] * 1024**2 + ) + end + end + end + + CLI::LogRuntimeArchiveMain.start(ARGV) + end +end diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index 7a033f137..2b695c7cc 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -24,7 +24,7 @@ def self.exit_on_failure? option :free_space_freed_limit, type: :numeric, default: 25_000, desc: "stop deleting files if available \ space is above this threshold (threshold in MB)" - default_task def watch(root_dir, target_dir) + default_task def watch(root_dir, target_dir) # rubocop:disable Metrics/AbcSize root_dir = validate_directory_exists(root_dir) target_dir = validate_directory_exists(target_dir) archiver = make_archiver(root_dir, target_dir) diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb new file mode 100644 index 000000000..725c5112f --- /dev/null +++ b/test/cli/test_log_runtime_archive_main.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require "syskit/test/self" +require "syskit/cli/log_runtime_archive_main" + +module Syskit + module CLI + describe LogRuntimeArchiveMain do + before do + @archive_dir = make_tmppath + @mocked_files_sizes = [] + + 10.times do |i| + ("/dev/tools/syskit/tmp/datasets" / i.to_s).write(i.to_s) + end + + @archiver = LogRuntimeArchiveMain.new( + "/dev/tools/syskit/tmp/datasets", + "/dev/tools/syskit/tmp/archive" + ) + end + + it "test cli" do + size_files = [6, 2, 1, 6, 7, 10, 3, 5, 8, 9] + mock_files_size(size_files) + mock_available_space(0.5) + CLI::LogRuntimeArchiveMain.start( + ["/tmp/archive", "/tmp/datasets", + "--free-space-low-limit", "345526079490", + "--free-space-freed-limit", "345526079500"] + ) + assert_deleted_files([0, 1, 2, 3]) + end + + def mock_files_size(sizes) + @mocked_files_sizes = sizes + @mocked_files_sizes.each_with_index do |size, i| + (@archive_dir / i.to_s).write(" " * size) + end + end + + def mock_available_space(total_disk_size) + flexmock(Sys::Filesystem) + .should_receive(:stat).with(@archive_dir) + .and_return do + flexmock( + bytes_free: total_disk_size + ) + end + end + + def assert_deleted_files(deleted_files) # rubocop:disable Metrics/AbcSize + if deleted_files.empty? + files = @archive_dir.each_child.select(&:file?) + assert_equal 10, files.size + else + (0..9).each do |i| + if deleted_files.include?(i) + refute (@archive_dir / i.to_s).exist?, + "#{i} was expected to be deleted, but has not been" + else + assert (@archive_dir / i.to_s).exist?, + "#{i} was expected to be present, but got deleted" + end + end + end + end + end + end +end From 264355fb904ba0db52f62ca4a4de5b6779fe2499 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 5 Feb 2024 14:13:39 -0300 Subject: [PATCH 124/260] feat: add log_runtime_archive_main call --- lib/syskit/scripts/log_runtime_archive.rb | 57 +---------------------- test/cli/test_log_runtime_archive_main.rb | 16 +++++-- 2 files changed, 13 insertions(+), 60 deletions(-) diff --git a/lib/syskit/scripts/log_runtime_archive.rb b/lib/syskit/scripts/log_runtime_archive.rb index 2b695c7cc..a996b2fc9 100644 --- a/lib/syskit/scripts/log_runtime_archive.rb +++ b/lib/syskit/scripts/log_runtime_archive.rb @@ -6,59 +6,6 @@ require "pathname" require "thor" require "syskit/cli/log_runtime_archive" +require "syskit/cli/log_runtime_archive_main" -# Command-line definition for the log-runtime-archive syskit subcommand -class CLI < Thor - def self.exit_on_failure? - true - end - - desc "watch", "watch a dataset root folder and archive the datasets" - option :period, - type: :numeric, default: 600, desc: "polling period in seconds" - option :max_size, - type: :numeric, default: 10_000, desc: "max log size in MB" - option :free_space_low_limit, - type: :numeric, default: 5_000, desc: "start deleting files if available \ - space is below this threshold (threshold in MB)" - option :free_space_freed_limit, - type: :numeric, default: 25_000, desc: "stop deleting files if available \ - space is above this threshold (threshold in MB)" - default_task def watch(root_dir, target_dir) # rubocop:disable Metrics/AbcSize - root_dir = validate_directory_exists(root_dir) - target_dir = validate_directory_exists(target_dir) - archiver = make_archiver(root_dir, target_dir) - loop do - archiver.ensure_free_space( - options[:free_space_low_limit] * 1000, - options[:free_space_freed_limit] * 1000 - ) - archiver.process_root_folder - - puts "Archived pending logs, sleeping #{options[:period]}s" - sleep options[:period] - end - end - - no_commands do - def validate_directory_exists(dir) - dir = Pathname.new(dir) - unless dir.directory? - raise ArgumentError, "#{dir} does not exist, or is not a directory" - end - - dir - end - - def make_archiver(root_dir, target_dir) - logger = Logger.new(STDOUT) - - Syskit::CLI::LogRuntimeArchive.new( - root_dir, target_dir, - logger: logger, max_archive_size: options[:max_size] * 1024**2 - ) - end - end -end - -CLI.start(ARGV) +Syskit::CLI::LogRuntimeArchiveMain.start(ARGV) diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 725c5112f..4c39b97d0 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -24,11 +24,8 @@ module CLI size_files = [6, 2, 1, 6, 7, 10, 3, 5, 8, 9] mock_files_size(size_files) mock_available_space(0.5) - CLI::LogRuntimeArchiveMain.start( - ["/tmp/archive", "/tmp/datasets", - "--free-space-low-limit", "345526079490", - "--free-space-freed-limit", "345526079500"] - ) + + call_command_line(@root, @archive_dir, 1e-3, 10 * 1e-3) assert_deleted_files([0, 1, 2, 3]) end @@ -65,6 +62,15 @@ def assert_deleted_files(deleted_files) # rubocop:disable Metrics/AbcSize end end end + + # Call 'archive' function instead of 'watch' to call archiver once + def call_command_line(root_path, archive_path, low_limit, freed_limit) + Syskit::CLI::CLIArchiveMain.start( + ["archive", root_path, archive_path, + "--free-space-low-limit", low_limit, + "--free-space-freed-limit", freed_limit] + ) + end end end end From 790a4e3bb884543941aa7389cf1302e79877ea47 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 5 Feb 2024 14:17:06 -0300 Subject: [PATCH 125/260] feat: add archive function to call unit tests once --- lib/syskit/cli/log_runtime_archive_main.rb | 35 ++++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive_main.rb b/lib/syskit/cli/log_runtime_archive_main.rb index c70bc92bd..a67b21cea 100644 --- a/lib/syskit/cli/log_runtime_archive_main.rb +++ b/lib/syskit/cli/log_runtime_archive_main.rb @@ -11,33 +11,42 @@ def self.exit_on_failure? true end - desc "watch", "watch a dataset root folder and archive the datasets" + desc "watch", "watch a dataset root folder and call archiver" + option :period, type: :numeric, default: 600, desc: "polling period in seconds" option :max_size, type: :numeric, default: 10_000, desc: "max log size in MB" - option :free_space_low_limit, - type: :numeric, default: 5_000, desc: "start deleting files if available \ - space is below this threshold (threshold in MB)" - option :free_space_freed_limit, - type: :numeric, default: 25_000, desc: "stop deleting files if available \ - space is above this threshold (threshold in MB)" default_task def watch(root_dir, target_dir) root_dir = validate_directory_exists(root_dir) target_dir = validate_directory_exists(target_dir) - archiver = make_archiver(root_dir, target_dir) loop do - archiver.ensure_free_space( - options[:free_space_low_limit] * 1000, - options[:free_space_freed_limit] * 1000 - ) - archiver.process_root_folder + archive(root_dir, target_dir) puts "Archived pending logs, sleeping #{options[:period]}s" sleep options[:period] end end + desc "archive", "archive the datasets and manages disk space" + option :max_size, + type: :numeric, default: 10_000, desc: "max log size in MB" + option :free_space_low_limit, + type: :numeric, default: 5_000, desc: "start deleting files if \ + available space is below this threshold (threshold in MB)" + option :free_space_freed_limit, + type: :numeric, default: 25_000, desc: "stop deleting files if \ + available space is above this threshold (threshold in MB)" + def archive(root_dir, target_dir) + archiver = make_archiver(root_dir, target_dir) + + archiver.ensure_free_space( + options[:free_space_low_limit] * 1000, + options[:free_space_freed_limit] * 1000 + ) + archiver.process_root_folder + end + no_commands do def validate_directory_exists(dir) dir = Pathname.new(dir) From c012355abf5ee4f84f464b7f0d3782e75069dbc4 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 5 Feb 2024 14:17:40 -0300 Subject: [PATCH 126/260] fix: minor fix --- lib/syskit/cli/log_runtime_archive_main.rb | 11 +++++++---- test/cli/test_log_runtime_archive.rb | 6 +++--- test/cli/test_log_runtime_archive_main.rb | 18 +++++++----------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive_main.rb b/lib/syskit/cli/log_runtime_archive_main.rb index a67b21cea..37d65f61c 100644 --- a/lib/syskit/cli/log_runtime_archive_main.rb +++ b/lib/syskit/cli/log_runtime_archive_main.rb @@ -1,12 +1,16 @@ # frozen_string_literal: true +# NOTE: this is NOT integrated in the Thor-based CLI to make it more independent +# (i.e. not depending on actually having Syskit installed) + require "pathname" require "thor" require "syskit/cli/log_runtime_archive" module Syskit module CLI - class LogRuntimeArchiveMain < Thor + # Command-line definition for the cli-archive-main syskit subcommand + class CLIArchiveMain < Thor def self.exit_on_failure? true end @@ -51,7 +55,8 @@ def archive(root_dir, target_dir) def validate_directory_exists(dir) dir = Pathname.new(dir) unless dir.directory? - raise ArgumentError, "#{dir} does not exist, or is not a directory" + raise ArgumentError, "#{dir} does not exist, or is not a "\ + "directory" end dir @@ -67,7 +72,5 @@ def make_archiver(root_dir, target_dir) end end end - - CLI::LogRuntimeArchiveMain.start(ARGV) end end diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index fbe9d318f..8b5b491e1 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -552,17 +552,17 @@ def mock_files_size(sizes) end end - def mock_available_space(total_disk_size) + def mock_available_space(total_available_disk_space) flexmock(Sys::Filesystem) .should_receive(:stat).with(@archive_dir) .and_return do flexmock( - bytes_free: total_disk_size + bytes_free: total_available_disk_space ) end end - def assert_deleted_files(deleted_files) + def assert_deleted_files(deleted_files) # rubocop:disable Metrics/AbcSize if deleted_files.empty? files = @archive_dir.each_child.select(&:file?) assert_equal 10, files.size diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 4c39b97d0..c0ae119e7 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -5,22 +5,18 @@ module Syskit module CLI - describe LogRuntimeArchiveMain do + describe CLIArchiveMain do before do + @root = make_tmppath @archive_dir = make_tmppath @mocked_files_sizes = [] - 10.times do |i| - ("/dev/tools/syskit/tmp/datasets" / i.to_s).write(i.to_s) - end + 10.times { |i| (@archive_dir / i.to_s).write(i.to_s) } - @archiver = LogRuntimeArchiveMain.new( - "/dev/tools/syskit/tmp/datasets", - "/dev/tools/syskit/tmp/archive" - ) + @archiver = LogRuntimeArchive.new(@root, @archive_dir) end - it "test cli" do + it "removes enough files to reach the freed limit" do size_files = [6, 2, 1, 6, 7, 10, 3, 5, 8, 9] mock_files_size(size_files) mock_available_space(0.5) @@ -36,12 +32,12 @@ def mock_files_size(sizes) end end - def mock_available_space(total_disk_size) + def mock_available_space(total_available_disk_space) flexmock(Sys::Filesystem) .should_receive(:stat).with(@archive_dir) .and_return do flexmock( - bytes_free: total_disk_size + bytes_free: total_available_disk_space ) end end From 2d8abd6ca295191880c2f616e418eb6f7c3169be Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 5 Feb 2024 17:48:44 -0300 Subject: [PATCH 127/260] feat: test cli fails if the directory does not exist --- test/cli/test_log_runtime_archive_main.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index c0ae119e7..8a49a67b4 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -6,14 +6,17 @@ module Syskit module CLI describe CLIArchiveMain do + it "raises ArgumentError if some of the directories do not exist" do + root = "make_tmppath" before do @root = make_tmppath @archive_dir = make_tmppath @mocked_files_sizes = [] - 10.times { |i| (@archive_dir / i.to_s).write(i.to_s) } - - @archiver = LogRuntimeArchive.new(@root, @archive_dir) + e = assert_raises ArgumentError do + call_command_line(root, @archive_dir, 1e3, 10 * 1e3) + end + assert_equal "#{root} does not exist, or is not a directory", e.message end it "removes enough files to reach the freed limit" do From edfd2b39f2bf2a7b65764480fb43748e65eef103 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 5 Feb 2024 17:49:52 -0300 Subject: [PATCH 128/260] fix: fix units matching --- lib/syskit/cli/log_runtime_archive_main.rb | 4 +- test/cli/test_log_runtime_archive_main.rb | 102 +++++++++++++-------- 2 files changed, 68 insertions(+), 38 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive_main.rb b/lib/syskit/cli/log_runtime_archive_main.rb index 37d65f61c..352c4e6b2 100644 --- a/lib/syskit/cli/log_runtime_archive_main.rb +++ b/lib/syskit/cli/log_runtime_archive_main.rb @@ -45,8 +45,8 @@ def archive(root_dir, target_dir) archiver = make_archiver(root_dir, target_dir) archiver.ensure_free_space( - options[:free_space_low_limit] * 1000, - options[:free_space_freed_limit] * 1000 + options[:free_space_low_limit] * 1e6, + options[:free_space_freed_limit] * 1e6 ) archiver.process_root_folder end diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 8a49a67b4..7aec19dfe 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -5,13 +5,10 @@ module Syskit module CLI + # Tests CLI command "archive" from syskit/cli/log_runtime_archive_main.rb describe CLIArchiveMain do it "raises ArgumentError if some of the directories do not exist" do root = "make_tmppath" - before do - @root = make_tmppath - @archive_dir = make_tmppath - @mocked_files_sizes = [] e = assert_raises ArgumentError do call_command_line(root, @archive_dir, 1e3, 10 * 1e3) @@ -19,44 +16,77 @@ module CLI assert_equal "#{root} does not exist, or is not a directory", e.message end - it "removes enough files to reach the freed limit" do - size_files = [6, 2, 1, 6, 7, 10, 3, 5, 8, 9] - mock_files_size(size_files) - mock_available_space(0.5) + describe "#ensure_free_space" do + before do + @root = make_tmppath + @archive_dir = make_tmppath + @mocked_files_sizes = [] - call_command_line(@root, @archive_dir, 1e-3, 10 * 1e-3) - assert_deleted_files([0, 1, 2, 3]) - end + 5.times { |i| (@archive_dir / i.to_s).write(i.to_s) } - def mock_files_size(sizes) - @mocked_files_sizes = sizes - @mocked_files_sizes.each_with_index do |size, i| - (@archive_dir / i.to_s).write(" " * size) + @archiver = LogRuntimeArchive.new(@root, @archive_dir) + end + + it "does nothing if there is enough free space" do + mock_available_space(2000) + call_command_line(@root, @archive_dir, 1000, 3000) + + assert_deleted_files([]) + end + + it "removes enough files to reach the freed limit" do + size_files = [750, 400, 900, 600, 700] + mock_files_size(size_files) + mock_available_space(700) # 700 MB + + call_command_line(@root, @archive_dir, 1000, 3000) + assert_deleted_files([0, 1, 2, 3]) end - end - def mock_available_space(total_available_disk_space) - flexmock(Sys::Filesystem) - .should_receive(:stat).with(@archive_dir) - .and_return do - flexmock( - bytes_free: total_available_disk_space - ) + it "stops removing files when there is no file in folder even if freed + limit is not achieved" do + size_files = Array.new(5, 100) + mock_files_size(size_files) + mock_available_space(500) + + call_command_line(@root, @archive_dir, 1000, 3000) + assert_deleted_files([0, 1, 2, 3, 4]) + end + + # Mock files sizes in bytes + # @param [Array] size of files in MB + def mock_files_size(sizes) + @mocked_files_sizes = sizes + @mocked_files_sizes.each_with_index do |size, i| + (@archive_dir / i.to_s).write(" " * size * 1e6) end - end + end + + # Mock total disk available space in bytes + # @param [Float] total_available_disk_space total available space in MB + def mock_available_space(total_available_disk_space) + flexmock(Sys::Filesystem) + .should_receive(:stat).with(@archive_dir) + .and_return do + flexmock( + bytes_free: total_available_disk_space * 1e6 + ) + end + end - def assert_deleted_files(deleted_files) # rubocop:disable Metrics/AbcSize - if deleted_files.empty? - files = @archive_dir.each_child.select(&:file?) - assert_equal 10, files.size - else - (0..9).each do |i| - if deleted_files.include?(i) - refute (@archive_dir / i.to_s).exist?, - "#{i} was expected to be deleted, but has not been" - else - assert (@archive_dir / i.to_s).exist?, - "#{i} was expected to be present, but got deleted" + def assert_deleted_files(deleted_files) # rubocop:disable Metrics/AbcSize + if deleted_files.empty? + files = @archive_dir.each_child.select(&:file?) + assert_equal 5, files.size + else + (0..4).each do |i| + if deleted_files.include?(i) + refute (@archive_dir / i.to_s).exist?, + "#{i} was expected to be deleted, but has not been" + else + assert (@archive_dir / i.to_s).exist?, + "#{i} was expected to be present, but got deleted" + end end end end From dd4369fe420c6177a029490244863f15cbdf70e6 Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Mon, 5 Feb 2024 17:50:24 -0300 Subject: [PATCH 129/260] fix: minor fix and directory validation --- lib/syskit/cli/log_runtime_archive.rb | 8 ++++---- lib/syskit/cli/log_runtime_archive_main.rb | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 49f2d9e2b..496ff5275 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -47,10 +47,10 @@ def process_root_folder # The method will check if there is enough space to save more log files # according to pre-established threshold. # - # @param [integer] free_space_low_limit: required free space threshold, at - # which the archiver starts deleting the oldest log files - # @param [integer] free_space_delete_until: post-deletion free space, at which - # the archiver stops deleting the oldest log files + # @param [integer] free_space_low_limit: required free space threshold in + # bytes, at which the archiver starts deleting the oldest log files + # @param [integer] free_space_delete_until: post-deletion free space in bytes, + # at which the archiver stops deleting the oldest log files def ensure_free_space(free_space_low_limit, free_space_delete_until) if free_space_low_limit > free_space_delete_until raise ArgumentError, diff --git a/lib/syskit/cli/log_runtime_archive_main.rb b/lib/syskit/cli/log_runtime_archive_main.rb index 352c4e6b2..b3c117e45 100644 --- a/lib/syskit/cli/log_runtime_archive_main.rb +++ b/lib/syskit/cli/log_runtime_archive_main.rb @@ -22,8 +22,6 @@ def self.exit_on_failure? option :max_size, type: :numeric, default: 10_000, desc: "max log size in MB" default_task def watch(root_dir, target_dir) - root_dir = validate_directory_exists(root_dir) - target_dir = validate_directory_exists(target_dir) loop do archive(root_dir, target_dir) @@ -42,6 +40,8 @@ def self.exit_on_failure? type: :numeric, default: 25_000, desc: "stop deleting files if \ available space is above this threshold (threshold in MB)" def archive(root_dir, target_dir) + root_dir = validate_directory_exists(root_dir) + target_dir = validate_directory_exists(target_dir) archiver = make_archiver(root_dir, target_dir) archiver.ensure_free_space( From c32a4ad64314e2c1b7e4bc04c6d0c8311ce5688a Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Tue, 6 Feb 2024 10:29:55 -0300 Subject: [PATCH 130/260] fix: update size of the created files and available disk space --- test/cli/test_log_runtime_archive_main.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 7aec19dfe..41b9e87d0 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -11,7 +11,7 @@ module CLI root = "make_tmppath" e = assert_raises ArgumentError do - call_command_line(root, @archive_dir, 1e3, 10 * 1e3) + call_command_line(root, @archive_dir, 1, 10) end assert_equal "#{root} does not exist, or is not a directory", e.message end @@ -28,28 +28,28 @@ module CLI end it "does nothing if there is enough free space" do - mock_available_space(2000) - call_command_line(@root, @archive_dir, 1000, 3000) + mock_available_space(200) + call_command_line(@root, @archive_dir, 100, 300) # 100 MB, 300 MB assert_deleted_files([]) end it "removes enough files to reach the freed limit" do - size_files = [750, 400, 900, 600, 700] + size_files = [75, 40, 90, 60, 70] mock_files_size(size_files) - mock_available_space(700) # 700 MB + mock_available_space(70) # 70 MB - call_command_line(@root, @archive_dir, 1000, 3000) + call_command_line(@root, @archive_dir, 100, 300) # 100 MB, 300 MB assert_deleted_files([0, 1, 2, 3]) end it "stops removing files when there is no file in folder even if freed limit is not achieved" do - size_files = Array.new(5, 100) + size_files = Array.new(5, 10) mock_files_size(size_files) - mock_available_space(500) + mock_available_space(80) # 80 MB - call_command_line(@root, @archive_dir, 1000, 3000) + call_command_line(@root, @archive_dir, 100, 300) # 100 MB, 300 MB assert_deleted_files([0, 1, 2, 3, 4]) end From b0f0af6de4610aa72a5d70d0fb4c6ea9a1c3c1bc Mon Sep 17 00:00:00 2001 From: Rayssa Soares Date: Tue, 6 Feb 2024 13:31:47 -0300 Subject: [PATCH 131/260] fix: fix extra spacing rubocop --- test/cli/test_log_runtime_archive_main.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 41b9e87d0..13f67b553 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -29,7 +29,7 @@ module CLI it "does nothing if there is enough free space" do mock_available_space(200) - call_command_line(@root, @archive_dir, 100, 300) # 100 MB, 300 MB + call_command_line(@root, @archive_dir, 100, 300) # 100 MB, 300 MB assert_deleted_files([]) end From cfd54227287258351910befdf962b5f6a42aaeb3 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 19 Feb 2024 21:22:09 -0300 Subject: [PATCH 132/260] fix: port collision in tests --- test/cli/test_gen_main.rb | 7 +++++-- test/cli/test_main.rb | 7 ++++--- test/roby_app/test_plugin.rb | 9 +++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/test/cli/test_gen_main.rb b/test/cli/test_gen_main.rb index 97ffa6b77..c746ac084 100644 --- a/test/cli/test_gen_main.rb +++ b/test/cli/test_gen_main.rb @@ -9,8 +9,11 @@ module CLI include Roby::Test::ArubaMinitest def assert_app_valid(*args) - syskit_run = run_command ["syskit", "run", *args].join(" ") - syskit_quit = run_command "syskit quit --retry" + port = roby_allocate_port + syskit_run = run_command( + ["syskit", "run", "--port=#{port}", *args].join(" ") + ) + syskit_quit = run_command "syskit quit --retry --host=localhost:#{port}" assert_command_stops syskit_run assert_command_stops syskit_quit diff --git a/test/cli/test_main.rb b/test/cli/test_main.rb index 02f8d26bb..05938f6d0 100644 --- a/test/cli/test_main.rb +++ b/test/cli/test_main.rb @@ -54,9 +54,10 @@ module CLI end it "forwards a Roby CLI command defined through Thor" do - run_cmd = run_command "roby run" - run_command_and_stop "roby wait" - run_command_and_stop "roby quit" + port = roby_allocate_port + run_cmd = run_command "roby run --port=#{port}" + run_command_and_stop "roby wait --host=localhost:#{port}" + run_command_and_stop "roby quit --host=localhost:#{port}" assert_command_stops run_cmd end end diff --git a/test/roby_app/test_plugin.rb b/test/roby_app/test_plugin.rb index d1a73f79d..61229bfdb 100644 --- a/test/roby_app/test_plugin.rb +++ b/test/roby_app/test_plugin.rb @@ -173,8 +173,7 @@ def perform_app_assertion(result) "models/pack/orogen/reload.orogen" copy_into_app "config/robots/reload_orogen.rb", "config/robots/default.rb" - pid = roby_app_spawn "run", silent: true - interface = assert_roby_app_is_running(pid) + pid, interface = roby_app_start "run", silent: true copy_into_app "models/pack/orogen/reload-2.orogen", "models/pack/orogen/reload.orogen" perform_app_assertion interface.unit_tests.orogen_deployment_exists? @@ -188,8 +187,7 @@ def perform_app_assertion(result) "models/compositions/reload_ruby_task.rb" copy_into_app "config/robots/reload_ruby_task.rb", "config/robots/default.rb" - pid = roby_app_spawn "run", silent: true - interface = assert_roby_app_is_running(pid) + pid, interface = roby_app_start "run", silent: true copy_into_app "models/compositions/reload_ruby_task-2.rb", "models/compositions/reload_ruby_task.rb" perform_app_assertion interface.unit_tests.orogen_deployment_exists? @@ -203,8 +201,7 @@ def perform_app_assertion(result) "models/pack/orogen/reload.orogen" copy_into_app "config/robots/reload_unmanaged_task.rb", "config/robots/default.rb" - pid = roby_app_spawn "run", silent: true - interface = assert_roby_app_is_running(pid) + pid, interface = roby_app_start "run", silent: true copy_into_app "models/pack/orogen/reload-2.orogen", "models/pack/orogen/reload.orogen" perform_app_assertion interface.unit_tests.orogen_deployment_exists? From f81ba325aa1ba246d6966a6ce86e4c01cff2ffad Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 3 Mar 2024 07:29:20 -0300 Subject: [PATCH 133/260] fix: name of the CLI class for `archive` The name in scripts/log_runtime_archive_main did not match the definition in cli/log_runtime_archive_main. Pick the name that maches the file name. --- lib/syskit/cli/log_runtime_archive_main.rb | 2 +- test/cli/test_log_runtime_archive_main.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive_main.rb b/lib/syskit/cli/log_runtime_archive_main.rb index b3c117e45..b57ffd361 100644 --- a/lib/syskit/cli/log_runtime_archive_main.rb +++ b/lib/syskit/cli/log_runtime_archive_main.rb @@ -10,7 +10,7 @@ module Syskit module CLI # Command-line definition for the cli-archive-main syskit subcommand - class CLIArchiveMain < Thor + class LogRuntimeArchiveMain < Thor def self.exit_on_failure? true end diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 13f67b553..8bd75e5f6 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -6,7 +6,7 @@ module Syskit module CLI # Tests CLI command "archive" from syskit/cli/log_runtime_archive_main.rb - describe CLIArchiveMain do + describe LogRuntimeArchiveMain do it "raises ArgumentError if some of the directories do not exist" do root = "make_tmppath" @@ -94,7 +94,7 @@ def assert_deleted_files(deleted_files) # rubocop:disable Metrics/AbcSize # Call 'archive' function instead of 'watch' to call archiver once def call_command_line(root_path, archive_path, low_limit, freed_limit) - Syskit::CLI::CLIArchiveMain.start( + LogRuntimeArchiveMain.start( ["archive", root_path, archive_path, "--free-space-low-limit", low_limit, "--free-space-freed-limit", freed_limit] From 7aff3d9d079760f72f68350eac722429cd497cb5 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 3 Mar 2024 07:42:00 -0300 Subject: [PATCH 134/260] chore: move all tests for the `archive` subcommand of log_runtime_archive under a single describe --- test/cli/test_log_runtime_archive_main.rb | 50 ++++++++++++++--------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 8bd75e5f6..505bf040b 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -7,29 +7,41 @@ module Syskit module CLI # Tests CLI command "archive" from syskit/cli/log_runtime_archive_main.rb describe LogRuntimeArchiveMain do - it "raises ArgumentError if some of the directories do not exist" do - root = "make_tmppath" - - e = assert_raises ArgumentError do - call_command_line(root, @archive_dir, 1, 10) + describe "#watch" do + before do + @root = make_tmppath + @archive_dir = make_tmppath end - assert_equal "#{root} does not exist, or is not a directory", e.message end - describe "#ensure_free_space" do + describe "#archive" do before do @root = make_tmppath @archive_dir = make_tmppath @mocked_files_sizes = [] 5.times { |i| (@archive_dir / i.to_s).write(i.to_s) } + end - @archiver = LogRuntimeArchive.new(@root, @archive_dir) + it "raises ArgumentError if the source directory does not exist" do + e = assert_raises ArgumentError do + call_archive("/does/not/exist", @archive_dir, 1, 10) + end + assert_equal "/does/not/exist does not exist, or is not a directory", + e.message + end + + it "raises ArgumentError if the target directory does not exist" do + e = assert_raises ArgumentError do + call_archive(@root, "/does/not/exist", 1, 10) + end + assert_equal "/does/not/exist does not exist, or is not a directory", + e.message end it "does nothing if there is enough free space" do mock_available_space(200) - call_command_line(@root, @archive_dir, 100, 300) # 100 MB, 300 MB + call_archive(@root, @archive_dir, 100, 300) # 100 MB, 300 MB assert_deleted_files([]) end @@ -39,7 +51,7 @@ module CLI mock_files_size(size_files) mock_available_space(70) # 70 MB - call_command_line(@root, @archive_dir, 100, 300) # 100 MB, 300 MB + call_archive(@root, @archive_dir, 100, 300) # 100 MB, 300 MB assert_deleted_files([0, 1, 2, 3]) end @@ -49,7 +61,7 @@ module CLI mock_files_size(size_files) mock_available_space(80) # 80 MB - call_command_line(@root, @archive_dir, 100, 300) # 100 MB, 300 MB + call_archive(@root, @archive_dir, 100, 300) # 100 MB, 300 MB assert_deleted_files([0, 1, 2, 3, 4]) end @@ -90,15 +102,15 @@ def assert_deleted_files(deleted_files) # rubocop:disable Metrics/AbcSize end end end - end - # Call 'archive' function instead of 'watch' to call archiver once - def call_command_line(root_path, archive_path, low_limit, freed_limit) - LogRuntimeArchiveMain.start( - ["archive", root_path, archive_path, - "--free-space-low-limit", low_limit, - "--free-space-freed-limit", freed_limit] - ) + # Call 'archive' function instead of 'watch' to call archiver once + def call_archive(root_path, archive_path, low_limit, freed_limit) + LogRuntimeArchiveMain.start( + ["archive", root_path, archive_path, + "--free-space-low-limit", low_limit, + "--free-space-freed-limit", freed_limit] + ) + end end end end From 8767d770e8ebcc8ae2a2311b09c09472522b9956 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 3 Mar 2024 07:49:45 -0300 Subject: [PATCH 135/260] fix: make sure `watch` sets all default options for `archive` --- lib/syskit/cli/log_runtime_archive_main.rb | 6 ++ test/cli/test_log_runtime_archive_main.rb | 103 +++++++++++++-------- 2 files changed, 71 insertions(+), 38 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive_main.rb b/lib/syskit/cli/log_runtime_archive_main.rb index b57ffd361..14454bdd4 100644 --- a/lib/syskit/cli/log_runtime_archive_main.rb +++ b/lib/syskit/cli/log_runtime_archive_main.rb @@ -21,6 +21,12 @@ def self.exit_on_failure? type: :numeric, default: 600, desc: "polling period in seconds" option :max_size, type: :numeric, default: 10_000, desc: "max log size in MB" + option :free_space_low_limit, + type: :numeric, default: 5_000, desc: "start deleting files if \ + available space is below this threshold (threshold in MB)" + option :free_space_freed_limit, + type: :numeric, default: 25_000, desc: "stop deleting files if \ + available space is above this threshold (threshold in MB)" default_task def watch(root_dir, target_dir) loop do archive(root_dir, target_dir) diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 505bf040b..c8defe5ed 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -11,6 +11,33 @@ module CLI before do @root = make_tmppath @archive_dir = make_tmppath + + @mocked_files_sizes = [] + 5.times { |i| (@archive_dir / i.to_s).write(i.to_s) } + end + + it "calls archive with the specified period" do + size_files = [75, 40, 90, 60, 70] + mock_files_size(size_files) + mock_available_space(200) # 70 MB + + quit = Class.new(RuntimeError) + called = 0 + flexmock(LogRuntimeArchive) + .new_instances + .should_receive(:process_root_folder) + .pass_thru do + called += 1 + raise quit if called == 3 + end + + assert_raises(quit) do + LogRuntimeArchiveMain.start( + ["watch", @root, @archive_dir, "--period", 0.5] + ) + end + + assert called == 3 end end @@ -65,44 +92,6 @@ module CLI assert_deleted_files([0, 1, 2, 3, 4]) end - # Mock files sizes in bytes - # @param [Array] size of files in MB - def mock_files_size(sizes) - @mocked_files_sizes = sizes - @mocked_files_sizes.each_with_index do |size, i| - (@archive_dir / i.to_s).write(" " * size * 1e6) - end - end - - # Mock total disk available space in bytes - # @param [Float] total_available_disk_space total available space in MB - def mock_available_space(total_available_disk_space) - flexmock(Sys::Filesystem) - .should_receive(:stat).with(@archive_dir) - .and_return do - flexmock( - bytes_free: total_available_disk_space * 1e6 - ) - end - end - - def assert_deleted_files(deleted_files) # rubocop:disable Metrics/AbcSize - if deleted_files.empty? - files = @archive_dir.each_child.select(&:file?) - assert_equal 5, files.size - else - (0..4).each do |i| - if deleted_files.include?(i) - refute (@archive_dir / i.to_s).exist?, - "#{i} was expected to be deleted, but has not been" - else - assert (@archive_dir / i.to_s).exist?, - "#{i} was expected to be present, but got deleted" - end - end - end - end - # Call 'archive' function instead of 'watch' to call archiver once def call_archive(root_path, archive_path, low_limit, freed_limit) LogRuntimeArchiveMain.start( @@ -112,6 +101,44 @@ def call_archive(root_path, archive_path, low_limit, freed_limit) ) end end + + # Mock files sizes in bytes + # @param [Array] size of files in MB + def mock_files_size(sizes) + @mocked_files_sizes = sizes + @mocked_files_sizes.each_with_index do |size, i| + (@archive_dir / i.to_s).write(" " * size * 1e6) + end + end + + # Mock total disk available space in bytes + # @param [Float] total_available_disk_space total available space in MB + def mock_available_space(total_available_disk_space) + flexmock(Sys::Filesystem) + .should_receive(:stat).with(@archive_dir) + .and_return do + flexmock( + bytes_free: total_available_disk_space * 1e6 + ) + end + end + + def assert_deleted_files(deleted_files) # rubocop:disable Metrics/AbcSize + if deleted_files.empty? + files = @archive_dir.each_child.select(&:file?) + assert_equal 5, files.size + else + (0..4).each do |i| + if deleted_files.include?(i) + refute (@archive_dir / i.to_s).exist?, + "#{i} was expected to be deleted, but has not been" + else + assert (@archive_dir / i.to_s).exist?, + "#{i} was expected to be present, but got deleted" + end + end + end + end end end end From 7613b5b61008b4331642126428f6f334f56f712f Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 3 Mar 2024 07:55:40 -0300 Subject: [PATCH 136/260] fix: do not use `e` notation when doing integer math NUMBEReEXP creates a float, which is not what we want. --- lib/syskit/cli/log_runtime_archive_main.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive_main.rb b/lib/syskit/cli/log_runtime_archive_main.rb index 14454bdd4..787bdc08f 100644 --- a/lib/syskit/cli/log_runtime_archive_main.rb +++ b/lib/syskit/cli/log_runtime_archive_main.rb @@ -51,8 +51,8 @@ def archive(root_dir, target_dir) archiver = make_archiver(root_dir, target_dir) archiver.ensure_free_space( - options[:free_space_low_limit] * 1e6, - options[:free_space_freed_limit] * 1e6 + options[:free_space_low_limit] * 1_000_000, + options[:free_space_freed_limit] * 1_000_000 ) archiver.process_root_folder end From 91a701d394f2d2f139e9cd5b9e755d13c46cb1f6 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 3 Mar 2024 08:20:31 -0300 Subject: [PATCH 137/260] fix: test for sleep duration in `log_runtime_archive watch` test --- test/cli/test_log_runtime_archive_main.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index c8defe5ed..066799ece 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -17,8 +17,7 @@ module CLI end it "calls archive with the specified period" do - size_files = [75, 40, 90, 60, 70] - mock_files_size(size_files) + mock_files_size([]) mock_available_space(200) # 70 MB quit = Class.new(RuntimeError) @@ -31,6 +30,7 @@ module CLI raise quit if called == 3 end + tic = Time.now assert_raises(quit) do LogRuntimeArchiveMain.start( ["watch", @root, @archive_dir, "--period", 0.5] @@ -38,6 +38,7 @@ module CLI end assert called == 3 + assert_operator(Time.now - tic, :>, 0.9) end end From a99bdb1a44720d39cc50db1fc39767fe53ae74ce Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 3 Mar 2024 08:21:15 -0300 Subject: [PATCH 138/260] fix: immediately re-run `archive` on ENOSPC in `log_runtime_archive watch` Free space management is mandatory, we therefore will free up some space on the archive folder and re-run the archiving. --- lib/syskit/cli/log_runtime_archive_main.rb | 6 +++++- test/cli/test_log_runtime_archive_main.rb | 25 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/syskit/cli/log_runtime_archive_main.rb b/lib/syskit/cli/log_runtime_archive_main.rb index 787bdc08f..3404b29e1 100644 --- a/lib/syskit/cli/log_runtime_archive_main.rb +++ b/lib/syskit/cli/log_runtime_archive_main.rb @@ -29,7 +29,11 @@ def self.exit_on_failure? available space is above this threshold (threshold in MB)" default_task def watch(root_dir, target_dir) loop do - archive(root_dir, target_dir) + begin + archive(root_dir, target_dir) + rescue Errno::ENOSPC + next + end puts "Archived pending logs, sleeping #{options[:period]}s" sleep options[:period] diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 066799ece..0901205db 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -40,6 +40,31 @@ module CLI assert called == 3 assert_operator(Time.now - tic, :>, 0.9) end + + it "retries on ENOSPC" do + mock_files_size([]) + mock_available_space(200) # 70 MB + + quit = Class.new(RuntimeError) + called = 0 + flexmock(LogRuntimeArchive) + .new_instances + .should_receive(:process_root_folder) + .pass_thru do + called += 1 + raise quit if called == 3 + + raise Errno::ENOSPC + end + + tic = Time.now + assert_raises(quit) do + LogRuntimeArchiveMain.start( + ["watch", @root, @archive_dir, "--period", 0.5] + ) + end + assert_operator(Time.now - tic, :<, 1) + end end describe "#archive" do From 66eb9c304d74bdf4f8681852433ddde7f8364272 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 13 Mar 2024 10:10:01 -0300 Subject: [PATCH 139/260] fix: do not "eat" exceptions in the log archiving process The current scheme was trying very hard to compress other files, assuming that the error would be file-specific. Turns out that this case does not happen (at least, did not happen so far) but it's very common to have a fatal error (think: no space on disk). --- lib/syskit/cli/log_runtime_archive.rb | 21 +++++++++++---------- test/cli/test_log_runtime_archive.rb | 16 +++++++++++----- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 496ff5275..b6beeb7e6 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -5,6 +5,9 @@ module Syskit module CLI + # Exception raised when the zstd subprocess returns an error + class CompressionFailed < RuntimeError; end + # Implementation of the `syskit log-runtime-archive` tool # # The tool archives Syskit log directories into tar archives in realtime, @@ -170,22 +173,20 @@ def self.add_to_archive(archive_io, child_path, logger: null_logger) data_pos = archive_io.tell exit_status = write_compressed_data(child_path, archive_io) - if exit_status.success? - add_to_archive_commit( - archive_io, child_path, start_pos, data_pos, stat - ) - child_path.unlink - true - else - add_to_archive_rollback(archive_io, start_pos, logger: logger) - false + unless exit_status.success? + raise CompressionFailed, "compression failed for #{child_path}" end + + add_to_archive_commit( + archive_io, child_path, start_pos, data_pos, stat + ) + child_path.unlink rescue Exception => e # rubocop:disable Lint/RescueException Roby.display_exception(STDOUT, e) if start_pos add_to_archive_rollback(archive_io, start_pos, logger: logger) end - false + raise end # Finalize appending a file in the archive diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 8b5b491e1..2717a2a47 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -87,8 +87,7 @@ module CLI refute blo.exist? end - it "restores the file as it was and keeps the input file if zstd fails "\ - "but continues with other files" do + it "restores the file as it was and keeps the input file if zstd fails" do bla = make_in_file "bla.txt", "bla" blo = make_in_file "blo.txt", "blo" bli = make_in_file "bli.txt", "bli" @@ -98,7 +97,11 @@ module CLI FlexMock.use(Process) do |mock| mock.should_receive(:waitpid2).once .and_return([10, flexmock(success?: false)]) - refute LogRuntimeArchive.add_to_archive(archive_io, blo) + + flexmock(Roby).should_receive(:display_exception).once + assert_raises(CompressionFailed) do + LogRuntimeArchive.add_to_archive(archive_io, blo) + end end assert LogRuntimeArchive.add_to_archive(archive_io, bli) end @@ -118,13 +121,16 @@ module CLI blo = make_in_file "blo.txt", "blo" bli = make_in_file "bli.txt", "bli" + exception_m = Class.new(RuntimeError) @archive_path.open("w") do |archive_io| assert LogRuntimeArchive.add_to_archive(archive_io, bla) FlexMock.use(Process) do |mock| mock.should_receive(:waitpid2).once - .and_raise(Exception.new) + .and_raise(exception_m.new) flexmock(Roby).should_receive(:display_exception).once - refute LogRuntimeArchive.add_to_archive(archive_io, blo) + assert_raises(exception_m) do + LogRuntimeArchive.add_to_archive(archive_io, blo) + end end assert LogRuntimeArchive.add_to_archive(archive_io, bli) end From f8fc72803b322bbe1048f2331c04ca6867a1d3f3 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 13 Mar 2024 09:50:44 -0300 Subject: [PATCH 140/260] fix: use bytes_available instead of bytes_free to determine free space The former is the total amount free on disk, the latter takes into account space reserved for priviledged users (e.g. root) --- lib/syskit/cli/log_runtime_archive.rb | 2 +- test/cli/test_log_runtime_archive.rb | 2 +- test/cli/test_log_runtime_archive_main.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/syskit/cli/log_runtime_archive.rb b/lib/syskit/cli/log_runtime_archive.rb index 496ff5275..18efed561 100644 --- a/lib/syskit/cli/log_runtime_archive.rb +++ b/lib/syskit/cli/log_runtime_archive.rb @@ -59,7 +59,7 @@ def ensure_free_space(free_space_low_limit, free_space_delete_until) end stat = Sys::Filesystem.stat(@target_dir) - available_space = stat.bytes_free + available_space = stat.bytes_available return if available_space > free_space_low_limit diff --git a/test/cli/test_log_runtime_archive.rb b/test/cli/test_log_runtime_archive.rb index 8b5b491e1..6fcb203cc 100644 --- a/test/cli/test_log_runtime_archive.rb +++ b/test/cli/test_log_runtime_archive.rb @@ -557,7 +557,7 @@ def mock_available_space(total_available_disk_space) .should_receive(:stat).with(@archive_dir) .and_return do flexmock( - bytes_free: total_available_disk_space + bytes_available: total_available_disk_space ) end end diff --git a/test/cli/test_log_runtime_archive_main.rb b/test/cli/test_log_runtime_archive_main.rb index 0901205db..5ce39b0ed 100644 --- a/test/cli/test_log_runtime_archive_main.rb +++ b/test/cli/test_log_runtime_archive_main.rb @@ -144,7 +144,7 @@ def mock_available_space(total_available_disk_space) .should_receive(:stat).with(@archive_dir) .and_return do flexmock( - bytes_free: total_available_disk_space * 1e6 + bytes_available: total_available_disk_space * 1e6 ) end end From 501b475e9ea89d94406418e081db7399e79eecdd Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 13 Mar 2024 17:12:27 -0300 Subject: [PATCH 141/260] fix: runtime view if the logger::Logger model is not loaded --- lib/syskit/gui/runtime_state.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syskit/gui/runtime_state.rb b/lib/syskit/gui/runtime_state.rb index 8c2178d9b..5791f500e 100644 --- a/lib/syskit/gui/runtime_state.rb +++ b/lib/syskit/gui/runtime_state.rb @@ -325,7 +325,7 @@ def logger_task?(t) @logger_m ||= Syskit::TaskContext .find_model_from_orogen_name("logger::Logger") || false - t.kind_of?(@logger_m) + t.kind_of?(@logger_m) if @logger_m end def update_tasks_info From f3c300d83f82536a76abd77a5f2c0b2c70d7e041 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 18 Sep 2023 21:06:30 -0300 Subject: [PATCH 142/260] fix: jobs may not be action tasks --- lib/syskit/gui/batch_manager.rb | 2 +- lib/syskit/gui/job_status_display.rb | 30 +++++++++++++++------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/syskit/gui/batch_manager.rb b/lib/syskit/gui/batch_manager.rb index 0a3017a1b..cc3e4a4f7 100644 --- a/lib/syskit/gui/batch_manager.rb +++ b/lib/syskit/gui/batch_manager.rb @@ -96,7 +96,7 @@ def start_job(action_name, action_arguments) def create_new_job(action_name, arguments = {}) action_model = @syskit.actions.find { |m| m.name == action_name } unless action_model - raise ArgumentError, "no action named #{action_name} found" + ::Kernel.raise ArgumentError, "no action named #{action_name} found" end if action_model.arguments.empty? diff --git a/lib/syskit/gui/job_status_display.rb b/lib/syskit/gui/job_status_display.rb index 8d02d620f..c6d874392 100644 --- a/lib/syskit/gui/job_status_display.rb +++ b/lib/syskit/gui/job_status_display.rb @@ -144,22 +144,24 @@ def connect_to_hooks @batch_manager.process end end - ui_restart.connect(SIGNAL("clicked()")) do - arguments = job.action_arguments.dup - arguments.delete(:job_id) - if @batch_manager.create_new_job(job.action_name, arguments) - @batch_manager.drop_job(self) - if @actions_immediate - @batch_manager.process + if job.action_name + ui_restart.connect(SIGNAL("clicked()")) do + arguments = job.action_arguments.dup + arguments.delete(:job_id) + if @batch_manager.create_new_job(job.action_name, arguments) + @batch_manager.drop_job(self) + if @actions_immediate + @batch_manager.process + end end end - end - ui_start.connect(SIGNAL("clicked()")) do - arguments = job.action_arguments.dup - arguments.delete(:job_id) - if @batch_manager.create_new_job(job.action_name, arguments) - if @actions_immediate - @batch_manager.process + ui_start.connect(SIGNAL("clicked()")) do + arguments = job.action_arguments.dup + arguments.delete(:job_id) + if @batch_manager.create_new_job(job.action_name, arguments) + if @actions_immediate + @batch_manager.process + end end end end From 382344038393816fec6fcffac7608c24e39f6d27 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 13 Mar 2024 17:13:03 -0300 Subject: [PATCH 143/260] fix: register "simulated" tasks on the corba naming services when in `syskit run` These are the tasks that come from `syskit run --sim`, which we definitely want to see. --- lib/syskit/roby_app/configuration.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/syskit/roby_app/configuration.rb b/lib/syskit/roby_app/configuration.rb index 4be2c359b..312cccc4c 100644 --- a/lib/syskit/roby_app/configuration.rb +++ b/lib/syskit/roby_app/configuration.rb @@ -414,7 +414,8 @@ def sim_process_server_config_for(name) ) register_process_server( sim_name, mng, - logging_enabled: false, register_on_name_server: false + logging_enabled: false, + register_on_name_server: !app.testing? ) end process_server_config_for(sim_name) From f2ced1efc8e4d14d4e181e630fd319aef9030a70 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Wed, 20 Mar 2024 20:03:48 -0300 Subject: [PATCH 144/260] fix: disable rubocop linting --- lib/syskit/gui/job_status_display.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syskit/gui/job_status_display.rb b/lib/syskit/gui/job_status_display.rb index c6d874392..c35f88832 100644 --- a/lib/syskit/gui/job_status_display.rb +++ b/lib/syskit/gui/job_status_display.rb @@ -137,7 +137,7 @@ def mouseReleaseEvent(event) end signals "clicked()" - def connect_to_hooks + def connect_to_hooks # rubocop:disable Metrics/PerceivedComplexity ui_drop.connect(SIGNAL("clicked()")) do @batch_manager.drop_job(self) if @actions_immediate From 30bd337791be5382ec3be672e9cdca04caa8e720 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 26 Mar 2024 09:46:15 -0300 Subject: [PATCH 145/260] fix: do not require roby/interface/async/interface explicitly --- test/gui/test_runtime_state.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/gui/test_runtime_state.rb b/test/gui/test_runtime_state.rb index 47e6537bc..857181396 100644 --- a/test/gui/test_runtime_state.rb +++ b/test/gui/test_runtime_state.rb @@ -6,7 +6,7 @@ require "roby/gui/exception_view" require "syskit/gui/state_label" require "syskit/gui/runtime_state" -require "roby/interface/async/interface" +require "roby/interface/async" module Syskit module GUI From be5c7fcbf709951dcd3338b4270afb2b613e6bac Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 18 Mar 2024 16:49:02 -0300 Subject: [PATCH 146/260] fix: handle resolution raising an error while all InstanceRequirements task have been removed This was triggering a (wrong) sanity check in the async resolution error handling, that was in the end killing syskit. --- .../apply_requirement_modifications.rb | 9 ++---- .../test_apply_requirement_modifications.rb | 28 ++++++++++++++++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/syskit/runtime/apply_requirement_modifications.rb b/lib/syskit/runtime/apply_requirement_modifications.rb index eb7aa7e1d..508af5cb0 100644 --- a/lib/syskit/runtime/apply_requirement_modifications.rb +++ b/lib/syskit/runtime/apply_requirement_modifications.rb @@ -113,13 +113,8 @@ def syskit_apply_async_resolution_results end nil rescue ::Exception => e # rubocop:disable Lint/RescueException - if running_requirement_tasks.empty? - add_framework_error(e, "deployment error without "\ - "requirement tasks") - else - running_requirement_tasks.each do |t| - t.failed_event.emit(e) - end + running_requirement_tasks.each do |t| + t.failed_event.emit(e) end e end diff --git a/test/runtime/test_apply_requirement_modifications.rb b/test/runtime/test_apply_requirement_modifications.rb index a41939554..bd45b0525 100644 --- a/test/runtime/test_apply_requirement_modifications.rb +++ b/test/runtime/test_apply_requirement_modifications.rb @@ -44,7 +44,33 @@ module Runtime execute { Runtime.apply_requirement_modifications(plan) } assert_resolution_cancelled do - execute { plan.unmark_permanent_task(requirement_task) } + expect_execution do + plan.unmark_permanent_task(requirement_task) + requirement_task.planning_task.stop! + end.to { have_error_matching Roby::PlanningFailedError.match } + end + + refute plan.syskit_current_resolution + end + + it "ignores resolution errors if all requirements have been "\ + "stopped in the meantime" do + cmp_m = Composition.new_submodel + requirement_task = + plan.add_permanent_task(cmp_m.to_instance_requirements.as_plan) + execute { requirement_task.planning_task.start! } + execute { Runtime.apply_requirement_modifications(plan) } + + error_m = Class.new(RuntimeError) + flexmock(plan.syskit_current_resolution) + .should_receive(:apply).and_raise(error_m) + + assert_resolution_cancelled do + expect_execution do + plan.unmark_permanent_task(requirement_task) + requirement_task.planning_task.stop! + end.to { have_error_matching Roby::PlanningFailedError.match } + Runtime.apply_requirement_modifications(plan) end refute plan.syskit_current_resolution From e344f428a1f07c3fef41ae745b2fe9a4125a3992 Mon Sep 17 00:00:00 2001 From: Jhonas Date: Tue, 9 Apr 2024 10:43:15 -0300 Subject: [PATCH 147/260] feat: take a class when creating a data_reader The klass argument allows the user to define their own class that subclasses BoundOutputReader. This way the user can create their own behavior on top of a data reader without needing to create the same interface as a DynamicPortBinding. --- lib/syskit/models/component.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/syskit/models/component.rb b/lib/syskit/models/component.rb index 7602f5cd4..6c0685cae 100644 --- a/lib/syskit/models/component.rb +++ b/lib/syskit/models/component.rb @@ -1311,14 +1311,14 @@ def match # data_reader some_child.out_port, as: 'pose' # # @return [DynamicPortBinding::BoundOutputReader] - def data_reader(port, as:, **policy) + def data_reader(port, as:, klass: BoundOutputReader, args: {}, **policy) port = DynamicPortBinding.create(port) unless port.output? raise ArgumentError, "expected an output port, but #{port} seems to be an input" end - data_readers[as] = port.to_bound_data_accessor(as, self, **policy) + klass.new(name, component_model, port, **args, **policy) end # The data writers defined on this task, as a mapping from the writer's From 42b972a1172bda2c4f2efd59b8177665a287414d Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 9 Apr 2024 11:12:56 -0300 Subject: [PATCH 148/260] fix: resolve keyword arguments warnings --- test/roby_app/test_rest_api.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/roby_app/test_rest_api.rb b/test/roby_app/test_rest_api.rb index 7f7fc3f89..1474b712d 100644 --- a/test/roby_app/test_rest_api.rb +++ b/test/roby_app/test_rest_api.rb @@ -135,7 +135,7 @@ def setup_deployed_task(deployed_task) it "returns the deployments as registered in Syskit" do configured_deployment = Models::ConfiguredDeployment.new( "localhost", @deployment_m, - Hash["test_task" => "mapped_test_task"], "test_deployment", {} + Hash["test_task" => "mapped_test_task"], "test_deployment" ) @configured_deployments << configured_deployment expected = Hash[ @@ -156,7 +156,7 @@ def setup_deployed_task(deployed_task) it "returns a deployment type of 'orocos' for an orocos remote process server" do configured_deployment = Models::ConfiguredDeployment.new( - "localhost", @deployment_m, Hash[], "test_deployment", {} + "localhost", @deployment_m, Hash[], "test_deployment" ) @configured_deployments << configured_deployment assert_equal "orocos", @@ -165,7 +165,7 @@ def setup_deployed_task(deployed_task) it "returns a deployment type of 'unmanaged' for an unmanaged task" do configured_deployment = Models::ConfiguredDeployment.new( - "unmanaged_tasks", @deployment_m, Hash[], "test_deployment", {} + "unmanaged_tasks", @deployment_m, Hash[], "test_deployment" ) @configured_deployments << configured_deployment assert_equal "unmanaged", @@ -174,7 +174,7 @@ def setup_deployed_task(deployed_task) it "ignores tasks whose process server type is not exported" do configured_deployment = Models::ConfiguredDeployment.new( - "something_else", @deployment_m, Hash[], "test_deployment", {} + "something_else", @deployment_m, Hash[], "test_deployment" ) @configured_deployments << configured_deployment assert_equal [], get_json("/deployments/registered")["registered_deployments"] @@ -182,7 +182,7 @@ def setup_deployed_task(deployed_task) it "reports if a deployment has not been created by the REST API" do configured_deployment = Models::ConfiguredDeployment.new( - "localhost", @deployment_m, Hash[], "test_deployment", {} + "localhost", @deployment_m, Hash[], "test_deployment" ) @configured_deployments << configured_deployment flexmock(RESTDeploymentManager).new_instances @@ -194,7 +194,7 @@ def setup_deployed_task(deployed_task) it "reports if a deployment has been created by the REST API" do configured_deployment = Models::ConfiguredDeployment.new( - "localhost", @deployment_m, Hash[], "test_deployment", {} + "localhost", @deployment_m, Hash[], "test_deployment" ) @configured_deployments << configured_deployment flexmock(RESTDeploymentManager).new_instances @@ -206,10 +206,10 @@ def setup_deployed_task(deployed_task) it "reports overriden deployments but not the tasks they are replaced by" do configured_deployment = Models::ConfiguredDeployment.new( - "unmanaged_tasks", @deployment_m, Hash[], "test_deployment", {} + "unmanaged_tasks", @deployment_m, Hash[], "test_deployment" ) overriden = Models::ConfiguredDeployment.new( - "localhost", @deployment_m, Hash[], "test_deployment", {} + "localhost", @deployment_m, Hash[], "test_deployment" ) @configured_deployments << configured_deployment mock = flexmock(RESTDeploymentManager).new_instances From e7b40ba14d19e1f349b14a4834bffe339686c3da Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 9 Apr 2024 11:57:54 -0300 Subject: [PATCH 149/260] fix: explicitly register the Syskit plugin on RobyAppHelpers RobyAppHelpers now restricts which plugins gets loaded to avoid effects due to the workspace. Explicitly require the syskit plugin in Syskit app tests. --- lib/syskit/test/roby_app_helpers.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/syskit/test/roby_app_helpers.rb b/lib/syskit/test/roby_app_helpers.rb index 3139ed3a3..b1483747f 100644 --- a/lib/syskit/test/roby_app_helpers.rb +++ b/lib/syskit/test/roby_app_helpers.rb @@ -8,6 +8,14 @@ module Test module RobyAppHelpers include Roby::Test::RobyAppHelpers + def setup + super + + register_plugin = + File.join(__dir__, "..", "roby_app", "register_plugin.rb") + register_roby_plugin(File.expand_path(register_plugin)) + end + def gen_app require "syskit/cli/gen_main" Dir.chdir(app_dir) { CLI::GenMain.start(["app", "--quiet"]) } From 111e8f558000620464ade06f911dcc3c19a0842d Mon Sep 17 00:00:00 2001 From: Jhonas Date: Tue, 9 Apr 2024 16:22:35 -0300 Subject: [PATCH 150/260] feat: inject class used when instantiating data_readers The injected class allows the user to override the default behavior of data readers. The class MUST be a subclass of Syskit::DynamicPortBinding::BountOutputReader. --- lib/syskit/models/component.rb | 19 +++++++++-- lib/syskit/models/dynamic_port_binding.rb | 11 ++++++- test/models/test_dynamic_port_binding.rb | 23 +++++++++++++ test/test_component.rb | 39 +++++++++++++++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/lib/syskit/models/component.rb b/lib/syskit/models/component.rb index 6c0685cae..9816308ff 100644 --- a/lib/syskit/models/component.rb +++ b/lib/syskit/models/component.rb @@ -1310,15 +1310,30 @@ def match # @example create a data reader for a composition child # data_reader some_child.out_port, as: 'pose' # + # @example inject another class to act as the output reader + # class Foo < Syskit::DynamicPortBinding::BoundOutputReader + # def read_new + # return unless (sample = super) + # + # sample + 42 + # end + # end + # + # data_reader some_child.out_port, as: "pose", klass: Foo + # # @return [DynamicPortBinding::BoundOutputReader] - def data_reader(port, as:, klass: BoundOutputReader, args: {}, **policy) + def data_reader( + port, as:, klass: Syskit::DynamicPortBinding::BoundOutputReader, **policy + ) port = DynamicPortBinding.create(port) unless port.output? raise ArgumentError, "expected an output port, but #{port} seems to be an input" end - klass.new(name, component_model, port, **args, **policy) + data_readers[as] = DynamicPortBinding::BoundOutputReader.new( + as, self, port, klass: klass, **policy + ) end # The data writers defined on this task, as a mapping from the writer's diff --git a/lib/syskit/models/dynamic_port_binding.rb b/lib/syskit/models/dynamic_port_binding.rb index 4c5b54868..bdae016e3 100644 --- a/lib/syskit/models/dynamic_port_binding.rb +++ b/lib/syskit/models/dynamic_port_binding.rb @@ -197,9 +197,18 @@ def initialize(name, component_model, *arguments, **kw_arguments) class BoundOutputReader < OutputReader include BoundAccessor + def initialize( + name, component_model, *arguments, + klass: Syskit::DynamicPortBinding::BoundOutputReader, **kw_arguments + ) + super(name, component_model, *arguments, **kw_arguments) + + @klass = klass + end + def instanciate(component, value_resolver: IdentityValueResolver.new) port_binding = @port_binding.instanciate - Syskit::DynamicPortBinding::BoundOutputReader.new( + @klass.new( name, component, port_binding, value_resolver: value_resolver, **policy ) diff --git a/test/models/test_dynamic_port_binding.rb b/test/models/test_dynamic_port_binding.rb index b971025cf..4ba5746ef 100644 --- a/test/models/test_dynamic_port_binding.rb +++ b/test/models/test_dynamic_port_binding.rb @@ -455,6 +455,29 @@ module Models end end + describe "injecting another instantiation class" do + before do + @port = DynamicPortBinding.new(flexmock, flexmock, + output: true, port_resolver: nil) + end + + it "instantiates using BoundOutputReader when klass is not passed" do + b = DynamicPortBinding::BoundOutputReader.new("bla", flexmock, @port) + result = b.instanciate(flexmock) + assert_kind_of(Syskit::DynamicPortBinding::BoundOutputReader, result) + end + + it "instantiates using the injected class" do + klass = Class.new Syskit::DynamicPortBinding::BoundOutputReader + + b = DynamicPortBinding::BoundOutputReader.new( + "bla", flexmock, @port, klass: klass + ) + result = b.instanciate(flexmock) + assert_kind_of(klass, result) + end + end + def make_resolver(type) app.default_loader.register_type_model(type) diff --git a/test/test_component.rb b/test/test_component.rb index 136c7ff2f..810c1f916 100644 --- a/test/test_component.rb +++ b/test/test_component.rb @@ -671,6 +671,21 @@ expect_execution.to { achieve { reader.connected? } } assert_equal task.out_port, reader.resolved_accessor.port end + + it "injects a new class instead of using BoundOutputReader" do + injected = Class.new Syskit::DynamicPortBinding::BoundOutputReader + + @support_task_m.data_reader( + @task_m.match.running.out_port, as: "test", klass: injected + ) + support_task = syskit_stub_deploy_configure_and_start(@support_task_m) + task = syskit_stub_deploy_configure_and_start(@task_m) + reader = support_task.test_reader + + assert reader.kind_of? injected + expect_execution.to { achieve { reader.connected? } } + assert_equal task.out_port, reader.resolved_accessor.port + end end describe Syskit::Component::DataAccessorInterface do @@ -837,6 +852,30 @@ syskit_stop(@task) end + it "uses injected class to update read samples" do + injected = Class.new Syskit::DynamicPortBinding::BoundOutputReader + injected.class_eval do + def read_new + return unless (sample = super) + + 22 + sample + end + end + + @task_m.data_reader( + @task_m.match.running.out_port, as: "test", klass: injected + ) + task = syskit_stub_deploy_configure_and_start(@task_m, remote_task: false) + + reader = task.test_reader + assert_kind_of injected, reader + expect_execution.to { achieve { reader.connected? } } + + expect_execution { syskit_write task.out_port, 20 }.to do + achieve { reader.read_new == 42 } + end + end + describe "deprecated string-based arguments" do before do @task = syskit_stub_and_deploy(@task_m) From 3820ba22c392cc8227bec8b2de4cbd872eace5d4 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 2 Apr 2024 13:12:51 -0300 Subject: [PATCH 151/260] chore: copy the runtime state view from gui/ into telemetry/ui/ This is only code copy and change of namespaces. No other changes. --- .rubocop_todo.yml | 47 ++ lib/syskit/telemetry/ui/app_start_dialog.rb | 100 +++ lib/syskit/telemetry/ui/batch_manager.rb | 242 +++++++ .../telemetry/ui/expanded_job_status.rb | 113 +++ lib/syskit/telemetry/ui/global_state_label.rb | 66 ++ lib/syskit/telemetry/ui/job_state_label.rb | 31 + lib/syskit/telemetry/ui/job_status_display.rb | 225 ++++++ .../telemetry/ui/logging_configuration.rb | 115 +++ .../ui/logging_configuration_item.rb | 75 ++ .../ui/logging_configuration_item_base.rb | 77 ++ .../telemetry/ui/logging_groups_item.rb | 55 ++ lib/syskit/telemetry/ui/ruby_item.rb | 41 ++ lib/syskit/telemetry/ui/runtime_state.rb | 665 ++++++++++++++++++ lib/syskit/telemetry/ui/state_label.rb | 207 ++++++ lib/syskit/telemetry/ui/widget_list.rb | 131 ++++ 15 files changed, 2190 insertions(+) create mode 100644 lib/syskit/telemetry/ui/app_start_dialog.rb create mode 100644 lib/syskit/telemetry/ui/batch_manager.rb create mode 100644 lib/syskit/telemetry/ui/expanded_job_status.rb create mode 100644 lib/syskit/telemetry/ui/global_state_label.rb create mode 100644 lib/syskit/telemetry/ui/job_state_label.rb create mode 100644 lib/syskit/telemetry/ui/job_status_display.rb create mode 100644 lib/syskit/telemetry/ui/logging_configuration.rb create mode 100644 lib/syskit/telemetry/ui/logging_configuration_item.rb create mode 100644 lib/syskit/telemetry/ui/logging_configuration_item_base.rb create mode 100644 lib/syskit/telemetry/ui/logging_groups_item.rb create mode 100644 lib/syskit/telemetry/ui/ruby_item.rb create mode 100644 lib/syskit/telemetry/ui/runtime_state.rb create mode 100644 lib/syskit/telemetry/ui/state_label.rb create mode 100644 lib/syskit/telemetry/ui/widget_list.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index b5f6e3277..8b61ad3b7 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -55,6 +55,7 @@ Layout/LineLength: - 'lib/syskit/gui/page_extension.rb' - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/state_label.rb' + - 'lib/syskit/telemetry/ui/**/*.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/instance_selection.rb' - 'lib/syskit/models/base.rb' @@ -234,6 +235,10 @@ Lint/AssignmentInCondition: - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/state_label.rb' - 'lib/syskit/gui/widget_list.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' + - 'lib/syskit/telemetry/ui/state_label.rb' + - 'lib/syskit/telemetry/ui/widget_list.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/instance_selection.rb' - 'lib/syskit/models/composition_child.rb' @@ -266,6 +271,7 @@ Lint/RescueException: Exclude: - 'lib/syskit/droby/v5/droby_dump.rb' - 'lib/syskit/gui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' - 'lib/syskit/gui/instanciate.rb' - 'lib/syskit/network_generation/async.rb' - 'lib/syskit/network_generation/dataflow_computation.rb' @@ -285,6 +291,7 @@ Lint/ShadowingOuterLocalVariable: - 'lib/syskit/gui/model_views/composition.rb' - 'lib/syskit/gui/model_views/data_service.rb' - 'lib/syskit/gui/runtime_state.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' - 'lib/syskit/models/deployment.rb' - 'lib/syskit/models/specialization_manager.rb' - 'test/models/test_specialization_manager.rb' @@ -296,6 +303,7 @@ Lint/SuppressedException: - 'lib/syskit/deployment.rb' - 'lib/syskit/droby/v5/droby_dump.rb' - 'lib/syskit/gui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' - 'lib/syskit/gui/page_extension.rb' - 'lib/syskit/remote_state_getter.rb' - 'lib/syskit/roby_app/plugin.rb' @@ -320,6 +328,10 @@ Lint/UnusedBlockArgument: - 'lib/syskit/gui/model_views/composition.rb' - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/widget_list.rb' + - 'lib/syskit/telemetry/ui/job_status_display.rb' + - 'lib/syskit/telemetry/ui/logging_configuration.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' + - 'lib/syskit/telemetry/ui/widget_list.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/models/composition_specialization.rb' - 'lib/syskit/models/specialization_manager.rb' @@ -346,6 +358,7 @@ Lint/UnusedMethodArgument: - 'lib/syskit/exceptions.rb' - 'lib/syskit/graphviz.rb' - 'lib/syskit/gui/expanded_job_status.rb' + - 'lib/syskit/telemetry/ui/expanded_job_status.rb' - 'lib/syskit/gui/model_views/type.rb' - 'lib/syskit/models/configured_deployment.rb' - 'lib/syskit/network_generation/dataflow_computation.rb' @@ -370,6 +383,7 @@ Lint/UselessAssignment: - 'lib/syskit/deployment.rb' - 'lib/syskit/gui/ide.rb' - 'lib/syskit/gui/state_label.rb' + - 'lib/syskit/telemetry/ui/state_label.rb' - 'lib/syskit/remote_state_getter.rb' - 'lib/syskit/roby_app/rest_api.rb' - 'lib/syskit/runtime/update_deployment_states.rb' @@ -449,6 +463,7 @@ Metrics/AbcSize: - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/testing.rb' - 'lib/syskit/gui/widget_list.rb' + - 'lib/syskit/telemetry/ui/**/*.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/instance_selection.rb' - 'lib/syskit/models/base.rb' @@ -585,6 +600,10 @@ Metrics/CyclomaticComplexity: - 'lib/syskit/gui/model_views/profile.rb' - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/testing.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/instanciate.rb' + - 'lib/syskit/telemetry/ui/job_status_display.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/instance_selection.rb' - 'lib/syskit/models/bound_data_service.rb' @@ -625,6 +644,7 @@ Metrics/MethodLength: Exclude: - 'lib/syskit/graphviz.rb' - 'lib/syskit/gui/runtime_state.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' - 'lib/syskit/models/composition.rb' - 'lib/syskit/network_generation/dataflow_computation.rb' - 'lib/syskit/network_generation/engine.rb' @@ -685,6 +705,8 @@ Metrics/PerceivedComplexity: - 'lib/syskit/gui/model_views/profile.rb' - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/testing.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/instance_selection.rb' - 'lib/syskit/models/bound_data_service.rb' @@ -754,6 +776,7 @@ Naming/MethodName: - 'lib/syskit/gui/ruby_item.rb' - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/widget_list.rb' + - 'lib/syskit/telemetry/ui/**/*.rb' - 'lib/syskit/network_generation/logger.rb' - 'lib/syskit/test/profile_assertions.rb' @@ -778,6 +801,9 @@ Naming/MethodParameterName: - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/testing.rb' - 'lib/syskit/gui/widget_list.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' + - 'lib/syskit/telemetry/ui/widget_list.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/models/base.rb' - 'lib/syskit/models/bound_data_service.rb' @@ -875,6 +901,7 @@ Style/Documentation: - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/testing.rb' - 'lib/syskit/gui/widget_list.rb' + - 'lib/syskit/telemetry/ui/**/*.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/models/base.rb' - 'lib/syskit/models/composition.rb' @@ -943,6 +970,7 @@ Style/FormatStringToken: Exclude: - 'lib/syskit/gui/ide.rb' - 'lib/syskit/gui/runtime_state.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' - 'lib/syskit/scripts/common.rb' # Offense count: 4 @@ -981,6 +1009,13 @@ Style/GuardClause: - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/state_label.rb' - 'lib/syskit/gui/widget_list.rb' + - 'lib/syskit/telemetry/ui/app_start_dialog.rb' + - 'lib/syskit/telemetry/ui/global_state_label.rb' + - 'lib/syskit/telemetry/ui/job_status_display.rb' + - 'lib/syskit/telemetry/ui/logging_configuration_item_base.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' + - 'lib/syskit/telemetry/ui/state_label.rb' + - 'lib/syskit/telemetry/ui/widget_list.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/instance_selection.rb' - 'lib/syskit/models/base.rb' @@ -1032,6 +1067,7 @@ Style/HashSyntax: - 'lib/syskit/coordination/models/task_extension.rb' - 'lib/syskit/deployment.rb' - 'lib/syskit/gui/logging_configuration.rb' + - 'lib/syskit/telemetry/ui/logging_configuration.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/models/specialization_manager.rb' - 'lib/syskit/roby_app/plugin.rb' @@ -1083,6 +1119,11 @@ Style/IfUnlessModifier: - 'lib/syskit/gui/model_views/task_context_base.rb' - 'lib/syskit/gui/runtime_state.rb' - 'lib/syskit/gui/widget_list.rb' + - 'lib/syskit/telemetry/ui/app_start_dialog.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/job_status_display.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' + - 'lib/syskit/telemetry/ui/widget_list.rb' - 'lib/syskit/instance_requirements.rb' - 'lib/syskit/instance_selection.rb' - 'lib/syskit/models/base.rb' @@ -1125,6 +1166,7 @@ Style/IfUnlessModifier: Style/MethodMissingSuper: Exclude: - 'lib/syskit/gui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' - 'lib/syskit/models/specialization_manager.rb' - 'lib/syskit/properties.rb' @@ -1132,6 +1174,7 @@ Style/MethodMissingSuper: Style/MissingRespondToMissing: Exclude: - 'lib/syskit/gui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' - 'lib/syskit/properties.rb' - 'lib/syskit/roby_app/single_file_dsl.rb' @@ -1151,6 +1194,7 @@ Style/PercentLiteralDelimiters: Exclude: - 'lib/syskit/gui/ide.rb' - 'lib/syskit/gui/runtime_state.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' - 'lib/syskit/network_generation/dataflow_computation.rb' - 'lib/syskit/scripts/common.rb' - 'test/test_component.rb' @@ -1166,6 +1210,8 @@ Style/RedundantBegin: - 'lib/syskit/deployment.rb' - 'lib/syskit/gui/batch_manager.rb' - 'lib/syskit/gui/logging_configuration.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/logging_configuration.rb' - 'lib/syskit/network_generation/dataflow_computation.rb' - 'lib/syskit/port.rb' - 'lib/syskit/roby_app/rest_api.rb' @@ -1182,6 +1228,7 @@ Style/RedundantBegin: Style/TrivialAccessors: Exclude: - 'lib/syskit/gui/batch_manager.rb' + - 'lib/syskit/telemetry/ui/batch_manager.rb' - 'lib/syskit/roby_app/configuration.rb' # Offense count: 2 diff --git a/lib/syskit/telemetry/ui/app_start_dialog.rb b/lib/syskit/telemetry/ui/app_start_dialog.rb new file mode 100644 index 000000000..0dcf91fff --- /dev/null +++ b/lib/syskit/telemetry/ui/app_start_dialog.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +module Syskit + module Telemetry + module UI + # A dialog that allows to get the parameters to start an app + class AppStartDialog < Qt::Dialog + # The combo box to choose the robot name + # + # @return [Qt::ComboBox] + attr_reader :robot_names + + # The checkbox allowing to choose whether the controller blocks + # should be executed or not + # + # @return [Qt::CheckBox] + attr_reader :start_controller + + # The checkbox allowing to choose whether Roby app should be run + # on single mode. For more information, check [Roby::Application#single] + # + # @return [Qt::CheckBox] + attr_reader :single + + # Text used to allow the user to not load any robot configuration + NO_ROBOT = " -- None -- " + + def initialize(names, parent = nil, default_robot_name: "default") + super(parent) + + self.window_title = "Start App" + + layout = Qt::VBoxLayout.new(self) + layout.add_widget(Qt::Label.new("Robot configuration to load:")) + layout.add_widget(@robot_names = Qt::ComboBox.new) + + robot_names.add_item NO_ROBOT + names.sort.each_with_index do |n, i| + robot_names.add_item(n) + if n == default_robot_name + robot_names.current_index = i + 1 + end + end + layout.add_widget(@start_controller = Qt::CheckBox.new("Start controller")) + start_controller.checked = true + + layout.add_widget(@single = Qt::CheckBox.new("Run on single mode")) + single.checked = false + + button_box = Qt::DialogButtonBox.new( + Qt::DialogButtonBox::Ok | Qt::DialogButtonBox::Cancel + ) + connect(button_box, SIGNAL("accepted()"), self, SLOT("accept()")) + connect(button_box, SIGNAL("rejected()"), self, SLOT("reject()")) + layout.add_widget(button_box) + end + + # The name of the selected robot + # + # @return [String] the robot name, or an empty string if no robot + # configuration should be loaded + def selected_name + txt = robot_names.current_text + if txt != NO_ROBOT + txt + else "" + end + end + + # Whether the controller should be started + # + # @return [Boolean] + def start_controller? + start_controller.checked? + end + + # Whether Roby app should be run on single mode + # + # @return [Boolean] + def single? + single.checked? + end + + # Executes a {AppStartDialog} in a modal way and returns the result + # + # @return [nil,(String,Boolean)] either nil if the dialog was + # rejected, or a robot name and a boolean indicating whether the + # controller blocks should be executed. The robot name can be + # empty to indicate that the dialog was accepted but no robot + # configuration should be loaded + def self.exec(names, parent = nil, default_robot_name: "default") + dialog = new(names, parent, default_robot_name: default_robot_name) + if Qt::Dialog::Accepted == dialog.exec + [dialog.selected_name, dialog.start_controller?, dialog.single?] + end + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/batch_manager.rb b/lib/syskit/telemetry/ui/batch_manager.rb new file mode 100644 index 000000000..bbb556fe0 --- /dev/null +++ b/lib/syskit/telemetry/ui/batch_manager.rb @@ -0,0 +1,242 @@ +# frozen_string_literal: true + +require "syskit/telemetry/ui/widget_list" + +module Syskit + module Telemetry + module UI + class BatchManager < WidgetList + def initialize(syskit, parent = nil) + super(parent) + @syskit = syskit + + @actions = Qt::Widget.new + actions_layout = Qt::HBoxLayout.new(@actions) + @process_btn = Qt::PushButton.new("Process") + @cancel_btn = Qt::PushButton.new("Cancel") + + actions_layout.add_widget(@process_btn) + actions_layout.add_widget(@cancel_btn) + add_widget(@actions, permanent: true) + + @process_btn.connect(SIGNAL(:clicked)) do + process + end + @cancel_btn.connect(SIGNAL(:clicked)) do + cancel + end + disable_actions + + @start_job = [] + @drop_job = [] + end + + def process + batch = @syskit.client.create_batch + @drop_job.each do |j| + batch.drop_job(j.job_id) + end + @start_job.each do |j| + arguments = j.action_arguments + arguments.delete(:job_id) + batch.start_job(j.action_name, arguments) + end + begin + batch.__process + rescue Exception => e + Roby.display_exception(STDOUT, e) + + Qt::MessageBox.critical( + self, "Syskit Process Batch", + "failed to process batch: #{e.message}, "\ + "see console output for more details" + ) + end + clear + end + + def cancel + clear + end + + def clear + clear_widgets + @start_job.clear + @drop_job.clear + disable_actions + end + + StartJob = Struct.new :action_name, :action_arguments + + def disable_actions + emit active(false) + @process_btn.enabled = false + @cancel_btn.enabled = false + end + + def enable_actions + emit active(true) + @process_btn.enabled = true + @cancel_btn.enabled = true + end + + signals "active(bool)" + + def drop_job(job_widget) + @drop_job << job_widget.job + add_before(Qt::Label.new("Drop #{job_widget.label}"), @actions) + enable_actions + end + + def start_job(action_name, action_arguments) + @start_job << StartJob.new(action_name, action_arguments) + add_before(Qt::Label.new("Start #{action_name}"), @actions) + enable_actions + end + + def create_new_job(action_name, arguments = {}) + action_model = @syskit.actions.find { |m| m.name == action_name } + unless action_model + ::Kernel.raise ArgumentError, "no action named #{action_name} found" + end + + if action_model.arguments.empty? + start_job(action_name, {}) + true + else + formatted_arguments = String.new + action_model.arguments.each do |arg| + default_arg = arguments.fetch( + arg.name.to_sym, arg.default + ) + has_default_arg = arguments.key?(arg.name.to_sym) || !arg.required? + + unless formatted_arguments.empty? + formatted_arguments << ",\n" + end + doc_lines = (arg.doc || "").split("\n") + formatted_arguments << "\n # #{doc_lines.join("\n # ")}\n" + if !has_default_arg + formatted_arguments << " #{arg.name}: " + elsif default_arg.nil? + formatted_arguments << " #{arg.name}: nil" + elsif default_arg.respond_to?(:name) && MetaRuby::Registration.accessible_by_name?(default_arg) + formatted_arguments << " #{arg.name}: #{default_arg.name}" + elsif as_string = ToString.convert(default_arg) + formatted_arguments << " #{arg.name}: #{as_string}" + elsif arg.required? + formatted_arguments << " #{arg.name}: " + else + formatted_arguments << " # #{arg.name}'s default argument cannot be handled by the IDE\n" + formatted_arguments << " # #{arg.name}: #{default_arg}" + + end + end + formatted_action = "#{action_name}!(\n#{formatted_arguments}\n)" + dialog = NewJobDialog.new(self, formatted_action) + if dialog.exec == Qt::Dialog::Accepted + action_name, action_options = dialog.result + start_job(action_name, action_options) + true + else false + end + end + end + + class ToString < BasicObject + def self.const_missing(const_name) + ::Object.const_get(const_name) + end + + def self.convert(obj) + if obj.kind_of?(Symbol) + ":#{obj}" + elsif obj.kind_of?(String) + "\"#{obj}\"" + else + as_string = obj.to_s + parser = new + begin + if parser.instance_eval(as_string) == obj + as_string + end + rescue Exception + end + end + end + end + + class NewJobDialog < Qt::Dialog + attr_reader :editor + + def initialize(parent = nil, text = "") + super(parent) + resize(800, 600) + + layout = Qt::VBoxLayout.new(self) + @error_message = Qt::Label.new(self) + @error_message.style_sheet = "QLabel { background-color: #ffb8b9; border: 1px solid #ff6567; padding: 5px; }" + @error_message.frame_style = Qt::Frame::StyledPanel + layout.add_widget(@error_message) + @error_message.hide + + @editor = Qt::TextEdit.new(self) + self.text = text + layout.add_widget editor + + buttons = Qt::DialogButtonBox.new(Qt::DialogButtonBox::Ok | Qt::DialogButtonBox::Cancel) + buttons.connect(SIGNAL("accepted()")) do + begin + @error_message.hide + @result = Parser.parse(self.text) + accept + rescue Exception => e + @error_message.text = e.message + @error_message.show + end + end + buttons.connect(SIGNAL("rejected()")) { reject } + layout.add_widget buttons + end + + def self.exec(parent, text) + new(parent, text).exec + end + + class Parser < BasicObject + def self.const_missing(const_name) + ::Object.const_get(const_name) + end + + def self.parse(text) + parser = new + parser.instance_eval(text) + parser.__result + end + + def method_missing(m, **options) + @method_name = m[0..-2] + @method_options = options + end + + def __result + [@method_name, @method_options] + end + end + + def result + @result + end + + def text=(text) + editor.plain_text = text + end + + def text + editor.to_plain_text + end + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/expanded_job_status.rb b/lib/syskit/telemetry/ui/expanded_job_status.rb new file mode 100644 index 000000000..815c64fd7 --- /dev/null +++ b/lib/syskit/telemetry/ui/expanded_job_status.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require "roby/gui/chronicle_widget" + +module Syskit + module Telemetry + module UI + # Detailed view on a job + # + # This is the widget displayed in Syskit IDE when a job is selected. It + # displays the task chronicle narrowed-down to the tasks involved in the + # job, as well as exceptions related to this task + class ExpandedJobStatus < WidgetList + # @return [Roby::GUI::ExceptionView] display of the exceptions that + # are related to this task + attr_reader :ui_exception_view + # @return [Roby::GUI::ChronicleWidget] the chronicle displaying + # the task states for the currently selected job, or for all tasks + # if there is no currently selected job + attr_reader :ui_chronicle + + # @return [JobStatusDisplay] the summary widget for the currently + # selected job, or nil if no job is selected + attr_reader :job_status + + def initialize(parent = nil) + super(parent, auto_resize: false) + + @ui_exception_view = Roby::GUI::ExceptionView.new + connect(ui_exception_view, SIGNAL("fileOpenClicked(const QUrl&)"), + self, SIGNAL("fileOpenClicked(const QUrl&)")) + @ui_chronicle = Roby::GUI::ChronicleWidget.new + ui_chronicle.show_mode = :in_range + ui_chronicle.reverse_sort = true + add_widget ui_exception_view + ui_exception_view.hide + add_widget ui_chronicle + @job_status = nil + end + + signals "fileOpenClicked(const QUrl&)" + + # Deselect the current job + # + # This updates the chronicle to show all tasks + def deselect + disconnect(self, SLOT("exceptionEvent()")) + @job_status = nil + ui_chronicle.clear_tasks_info + update_exceptions([]) + end + + # Select a given job + # + # It reduces the displayed information to only the information that + # involved the selected job + # + # @param [JobStatusDisplay] job_status the job status widget that + # represents the job to be selected + def select(job_status) + disconnect(self, SLOT("exceptionEvent()")) + @job_status = job_status + ui_chronicle.clear_tasks_info + update_exceptions(job_status.exceptions) + connect(job_status, SIGNAL("exceptionEvent()"), self, SLOT("exceptionEvent()")) + end + + # Add task and job info + def add_tasks_info(tasks_info, job_info) + ui_chronicle.add_tasks_info(tasks_info, job_info) + end + + # Set the current scheduler state + def scheduler_state=(state) + ui_chronicle.scheduler_state = state + end + + # Update the chronicle display + def update_chronicle + ui_chronicle.update_current_tasks + ui_chronicle.update + children_size_updated + end + + # Update the current time + # + # @param [Integer] cycle_index the index of the current Roby cycle + # @param [Time] cycle_time the time of the current Roby cycle + def update_time(cycle_index, cycle_time) + ui_chronicle.update_current_time(cycle_time) + end + + # Slot used to announce that the exceptions registerd on + # {#job_status} have changed + def exceptionEvent + update_exceptions(job_status.exceptions) + end + slots "exceptionEvent()" + + # Update the exception display to display the given exceptions + def update_exceptions(exceptions) + ui_exception_view.exceptions = exceptions.dup + if exceptions.empty? + ui_exception_view.hide + else + ui_exception_view.show + end + children_size_updated + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/global_state_label.rb b/lib/syskit/telemetry/ui/global_state_label.rb new file mode 100644 index 000000000..95d124659 --- /dev/null +++ b/lib/syskit/telemetry/ui/global_state_label.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Syskit + module Telemetry + module UI + # Representation of the state of the connected Roby instance + class GlobalStateLabel < StateLabel + # Actions that are shown when the context menu is activated + # + # @return [Array] the list of actions that can be + # performed on the remote Roby instance (e.g. start/stop ...) + attr_reader :actions + + # @param [Array] the list of actions that can be + # performed on the remote Roby instance + def initialize(actions: [], **options) + super(extra_style: "margin-left: 2px; margin-top: 2px; font-size: 10pt;", + rate_limited: true, **options) + @actions = actions + declare_state "STARTING", :blue + declare_state "RESTARTING", :blue + declare_state "CONNECTED", :green + declare_state "UNREACHABLE", :red + end + + # @api private + # + # Qt handler called when the context menu is activated + def contextMenuEvent(event) + unless actions.empty? + app_state_menu(event.global_pos) + event.accept + end + end + + # Execute the app state menu + def app_state_menu(global_pos) + unless actions.empty? + menu = Qt::Menu.new(self) + actions.each { |act| menu.add_action(act) } + menu.exec(global_pos) + true + end + end + + # @api private + # + # Qt handler called when the mouse is pressed + def mousePressEvent(event) + event.accept + end + + # @api private + # + # Qt handler called when the mouse is released + # + # It emits the 'clicked' signal + def mouseReleaseEvent(event) + emit clicked(event.global_pos) + event.accept + end + signals "clicked(QPoint)" + end + end + end +end diff --git a/lib/syskit/telemetry/ui/job_state_label.rb b/lib/syskit/telemetry/ui/job_state_label.rb new file mode 100644 index 000000000..f16adfba3 --- /dev/null +++ b/lib/syskit/telemetry/ui/job_state_label.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Syskit + module Telemetry + module UI + # A label that can be used to represent job states + # + # It [declares](StateLabel#declare_state) the known job states and + # assigns proper colors to it. + # + # @example + class JobStateLabel < StateLabel + def initialize(**options) + super + declare_default_color :red + declare_state Roby::Interface::JOB_PLANNING_READY.upcase, :blue + declare_state Roby::Interface::JOB_PLANNING.upcase, :blue + declare_state Roby::Interface::JOB_PLANNING_FAILED.upcase, :red + declare_state Roby::Interface::JOB_READY.upcase, :blue + declare_state Roby::Interface::JOB_STARTED.upcase, :green + declare_state Roby::Interface::JOB_SUCCESS.upcase, :grey + declare_state Roby::Interface::JOB_FAILED.upcase, :red + declare_state Roby::Interface::JOB_FINISHED.upcase, :grey + declare_state Roby::Interface::JOB_FINALIZED.upcase, :grey + + declare_state Roby::Interface::JOB_DROPPED.upcase, :grey + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/job_status_display.rb b/lib/syskit/telemetry/ui/job_status_display.rb new file mode 100644 index 000000000..68208c075 --- /dev/null +++ b/lib/syskit/telemetry/ui/job_status_display.rb @@ -0,0 +1,225 @@ +# frozen_string_literal: true + +require "syskit/telemetry/ui/state_label" +require "syskit/telemetry/ui/job_state_label" + +module Syskit + module Telemetry + module UI + class JobStatusDisplay < Qt::Widget + attr_reader :job + + attr_reader :ui_job_actions + attr_reader :ui_start + attr_reader :ui_restart + attr_reader :ui_drop + attr_reader :ui_clear + attr_reader :ui_state + attr_reader :exceptions + attr_reader :notifications + attr_reader :ui_notifications + + attr_predicate :show_actions?, true + + def initialize(job, batch_manager) + super(nil) + @batch_manager = batch_manager + @job = job + @exceptions = [] + @notifications = {} + @show_actions = true + + create_ui + connect_to_hooks + hide_job_actions + end + + INTERMEDIATE_TERMINAL_STATES = [ + Roby::Interface::JOB_SUCCESS.upcase.to_s, + Roby::Interface::JOB_DROPPED.upcase.to_s, + Roby::Interface::JOB_FAILED.upcase.to_s, + Roby::Interface::JOB_PLANNING_FAILED.upcase.to_s + ].freeze + + def label + "##{job.job_id} #{job.job_name}" + end + + def create_ui + self.focus_policy = Qt::ClickFocus + @ui_state = JobStateLabel.new name: label + if job.state + ui_state.update_state(job.state.upcase) + end + + @ui_job_actions = Qt::Widget.new(self) + hlayout = Qt::HBoxLayout.new(ui_job_actions) + @actions_buttons = Hash[ + "Drop" => Qt::PushButton.new("Drop", self), + "Restart" => Qt::PushButton.new("Restart", self), + "Start Again" => Qt::PushButton.new("Start Again", self), + "Clear" => Qt::PushButton.new("Clear", self) + ] + hlayout.add_widget(@ui_drop = @actions_buttons["Drop"]) + hlayout.add_widget(@ui_restart = @actions_buttons["Restart"]) + hlayout.add_widget(@ui_start = @actions_buttons["Start Again"]) + hlayout.add_widget(@ui_clear = @actions_buttons["Clear"]) + + ui_start.hide + ui_clear.hide + hlayout.set_contents_margins(0, 0, 0, 0) + + @ui_notifications = Qt::Label.new("", self) + ui_notifications.hide + + vlayout = Qt::VBoxLayout.new(self) + vlayout.add_widget ui_state + vlayout.add_widget ui_job_actions + vlayout.add_widget ui_notifications + end + + def keyPressEvent(event) + make_actions_immediate(event.key == Qt::Key_Control) + super + end + + def keyReleaseEvent(event) + make_actions_immediate(false) + super + end + + def make_actions_immediate(enable) + @actions_immediate = enable + if enable + @actions_buttons.each do |text, btn| + btn.text = "#{text} Now" + end + else + @actions_buttons.each do |text, btn| + btn.text = text + end + end + end + + def show_job_actions + ui_job_actions.show + s = size + s.height = size_hint.height + self.size = s + end + + def hide_job_actions + ui_job_actions.hide + s = size + s.height = size_hint.height + self.size = s + end + + def enterEvent(event) + super + if show_actions? + show_job_actions + self.focus = Qt::OtherFocusReason + end + end + + def leaveEvent(event) + super + if show_actions? + hide_job_actions + end + end + + def mousePressEvent(event) + emit clicked + event.accept + end + + def mouseReleaseEvent(event) + event.accept + end + signals "clicked()" + + def connect_to_hooks # rubocop:disable Metrics/PerceivedComplexity + ui_drop.connect(SIGNAL("clicked()")) do + @batch_manager.drop_job(self) + if @actions_immediate + @batch_manager.process + end + end + if job.action_name + ui_restart.connect(SIGNAL("clicked()")) do + arguments = job.action_arguments.dup + arguments.delete(:job_id) + if @batch_manager.create_new_job(job.action_name, arguments) + @batch_manager.drop_job(self) + if @actions_immediate + @batch_manager.process + end + end + end + ui_start.connect(SIGNAL("clicked()")) do + arguments = job.action_arguments.dup + arguments.delete(:job_id) + if @batch_manager.create_new_job(job.action_name, arguments) + if @actions_immediate + @batch_manager.process + end + end + end + end + ui_clear.connect(SIGNAL("clicked()")) do + unless job.active? + job.stop + emit clearJob + true + end + end + job.on_progress do |state| + update_state(state) + end + job.on_exception do |kind, exception| + exceptions << exception.exception + notify("exceptions", "#{exceptions.size} exceptions") + emit exceptionEvent + end + end + + def update_state(state) + if INTERMEDIATE_TERMINAL_STATES.include?(ui_state.current_state) + ui_state.update_state( + "#{ui_state.current_state}, + #{state.upcase}", + color: ui_state.current_color + ) + else + ui_state.update_state(state.upcase) + end + + if state == Roby::Interface::JOB_DROPPED + ui_drop.hide + ui_restart.hide + ui_start.show + ui_clear.show + elsif Roby::Interface.terminal_state?(state) + ui_drop.hide + ui_restart.hide + ui_start.show + ui_clear.show + end + end + + def notify(key, text) + notifications[key] = text + ui_notifications.show + ui_notifications.text = "#{notifications.values.join(', ')}" + end + + # Signal emitted when one exception got added at the end of + # {#exceptions} + signals "exceptionEvent()" + signals "clearJob()" + end + end + end +end diff --git a/lib/syskit/telemetry/ui/logging_configuration.rb b/lib/syskit/telemetry/ui/logging_configuration.rb new file mode 100644 index 000000000..fa232cd7d --- /dev/null +++ b/lib/syskit/telemetry/ui/logging_configuration.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +require "vizkit" +require "vizkit/vizkit_items" +require "vizkit/tree_view" +require "Qt4" +require "syskit/interface" +require "syskit/telemetry/ui/logging_configuration_item" +require "roby/interface/exceptions" + +module Syskit + module Telemetry + module UI + # A widget containing an editable TreeView to allow the user to + # manage basic Syskit's logging configuration + class LoggingConfiguration < Qt::Widget + attr_reader :model, :tree_view, :syskit, :item_name, :item_value, :pending_call + def initialize(syskit, parent = nil) + super(parent) + main_layout = Qt::VBoxLayout.new(self) + @tree_view = Qt::TreeView.new + + Vizkit.setup_tree_view tree_view + @model = Vizkit::VizkitItemModel.new + tree_view.setModel @model + main_layout.add_widget(tree_view) + tree_view.setColumnWidth(0, 200) + tree_view.style_sheet = <<~STYLESHEET + QTreeView { + background-color: rgb(255, 255, 219); + alternate-background-color: rgb(255, 255, 174); + color: rgb(0, 0, 0); + } + QTreeView:disabled { color: rgb(159, 158, 158); } + STYLESHEET + + @syskit = syskit + @timer = Qt::Timer.new + @timer.connect(SIGNAL("timeout()")) { refresh } + @timer.start 1500 + + refresh + end + + # Whether there is a refreshing call pending + def refreshing? + syskit.async_call_pending?(pending_call) + end + + # Fetches the current logging configuration from syskit's + # sync interface + def refresh + if syskit.reachable? + begin + return if refreshing? + + @pending_call = syskit.async_call ["syskit"], :logging_conf do |error, result| + if error.nil? + enabled true + update_model(result) + else + enabled false + end + end + rescue Roby::Interface::ComError + enabled false + end + else + enabled false + end + end + + # Expands the entire tree + def recursive_expand(item) + tree_view.expand(item.index) + (0...item.rowCount).each do |i| + recursive_expand(item.child(i)) + end + end + + # Changes the top most item in the tree state + # and makes it update its childs accordingly + def enabled(toggle) + @item_name&.enabled toggle + end + + # Updates the view model + def update_model(conf) + if @item_name.nil? + @item_name = LoggingConfigurationItem.new(conf, :accept => true) + @item_value = LoggingConfigurationItem.new(conf) + @item_value.setEditable true + @item_value.setText "" + @model.appendRow([@item_name, @item_value]) + recursive_expand(@item_name) + + @item_name.on_accept_changes do |new_conf| + begin + syskit.async_call ["syskit"], :update_logging_conf, new_conf do |error, result| + enabled false unless error.nil? + end + rescue Roby::Interface::ComError + enabled false + end + end + else + return if @item_name.modified? + + @item_name.update_conf(conf) + end + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/logging_configuration_item.rb b/lib/syskit/telemetry/ui/logging_configuration_item.rb new file mode 100644 index 000000000..a9a0e53b1 --- /dev/null +++ b/lib/syskit/telemetry/ui/logging_configuration_item.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require "vizkit" +require "Qt4" +require "syskit/telemetry/ui/logging_configuration_item_base" +require "syskit/telemetry/ui/logging_groups_item" + +module Syskit + module Telemetry + module UI + # A QStandardItem that displays a Sysit::ShellInterface::LoggingConfiguration + # in a tree view + class LoggingConfigurationItem < LoggingConfigurationItemBase + attr_reader :options + attr_reader :conf_logs_item_name + attr_reader :conf_logs_item_value + attr_reader :port_logs_item_name + attr_reader :port_logs_item_value + attr_reader :groups_item_name + attr_reader :groups_item_value + def initialize(logging_configuration, options = {}) + super(logging_configuration) + @options = options + setText "Logging Configuration" + + @conf_logs_item_name, @conf_logs_item_value = add_conf_item("Enable conf logs", + :conf_logs_enabled) + @port_logs_item_name, @port_logs_item_value = add_conf_item("Enable port logs", + :port_logs_enabled) + + @groups_item_name = LoggingGroupsItem.new(@current_model.groups, "Enable group") + @groups_item_value = Vizkit::VizkitItem.new("#{@current_model.groups.size} logging group(s)") + appendRow([@groups_item_name, @groups_item_value]) + end + + # Called when the user commit changes made to the model + def write + if column == 1 + i = index.sibling(row, 0) + return unless i.isValid + + item = i.model.itemFromIndex i + item.accept_changes + end + modified!(false) + end + + # Notify child to also accept user changes, + # updates internal copy of the logging configuration, and calls + # a block that should the data to the remote side + def accept_changes + super + @groups_item_name.accept_changes + @current_model.groups = @groups_item_name.current_model + @commit_block.call current_model + end + + # Updates the model with a new logging configuration + def update_conf(new_model) + @current_model = deep_copy(new_model) + @editing_model = deep_copy(new_model) + @groups_item_name.update_groups(@current_model.groups) + @groups_item_value.setText "#{@current_model.groups.size} logging group(s)" + model.layoutChanged + end + + # Sets the block to be called when the user accepts the changes + # made to the model + def on_accept_changes(&block) + @commit_block = block + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/logging_configuration_item_base.rb b/lib/syskit/telemetry/ui/logging_configuration_item_base.rb new file mode 100644 index 000000000..a368a783b --- /dev/null +++ b/lib/syskit/telemetry/ui/logging_configuration_item_base.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +require "vizkit" +require "Qt4" +require "syskit/telemetry/ui/ruby_item" + +module Syskit + module Telemetry + module UI + # Base class for most items in the LoggingConfiguration widget with + # common functionality + class LoggingConfigurationItemBase < Vizkit::VizkitItem + attr_reader :current_model + attr_reader :editing_model + def initialize(model) + super() + @current_model = deep_copy(model) + @editing_model = deep_copy(model) + end + + # Creates a marshallable deep copy of the object + def deep_copy(model) + Marshal.load(Marshal.dump(model)) + end + + # Adds a ruby primitive type to the tree view + def add_conf_item(label, accessor = nil) + item1 = Vizkit::VizkitItem.new(label) + item2 = RubyItem.new + + unless accessor.nil? + item2.getter do + @editing_model.method(accessor).call + end + + item2.setter do |value| + @editing_model.method("#{accessor}=".to_sym).call value + end + end + + appendRow([item1, item2]) + [item1, item2] + end + + def data(role = Qt::UserRole + 1) + if role == Qt::EditRole + Qt::Variant.from_ruby self + else + super + end + end + + # Updates view's sibling modified? state possibly rejecting changes + # made to the model + def modified!(value = true, items = [], update_parent = false) + super + reject_changes unless value + if column == 0 + i = index.sibling(row, 1) + if i.isValid + item = i.model.itemFromIndex i + item.modified!(value, items) + end + end + end + + def reject_changes + @editing_model = deep_copy(@current_model) + end + + def accept_changes + @current_model = deep_copy(@editing_model) + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/logging_groups_item.rb b/lib/syskit/telemetry/ui/logging_groups_item.rb new file mode 100644 index 000000000..61ecbbf20 --- /dev/null +++ b/lib/syskit/telemetry/ui/logging_groups_item.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require "vizkit" +require "Qt4" +require "syskit/telemetry/ui/logging_configuration_item_base" + +module Syskit + module Telemetry + module UI + # A QStandardItem to display a hash of Sysit::ShellInterface::LoggingGroup + # in a tree view + class LoggingGroupsItem < LoggingConfigurationItemBase + attr_reader :items_name, :items_value + def initialize(logging_groups, label = "") + super(logging_groups) + + @items_name = {} + @items_value = {} + + setText label + update_groups(logging_groups) + end + + # Updates the model according to a new hash + def update_groups(groups) + @current_model.keys.each do |key| + unless groups.key? key + group_row = @items_name[key].index.row + @items_name[key].clear + @items_value[key].clear + @items_name.delete key + @items_value.delete key + removeRow(group_row) + end + end + + @current_model = deep_copy(groups) + @editing_model = deep_copy(groups) + + @current_model.keys.each do |key| + unless @items_name.key? key + @items_name[key], @items_value[key] = add_conf_item(key) + @items_value[key].getter do + @editing_model[key].enabled + end + @items_value[key].setter do |value| + @editing_model[key].enabled = value + end + end + end + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/ruby_item.rb b/lib/syskit/telemetry/ui/ruby_item.rb new file mode 100644 index 000000000..2a876336e --- /dev/null +++ b/lib/syskit/telemetry/ui/ruby_item.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require "vizkit" +require "Qt4" +require "vizkit/vizkit_items" + +module Syskit + module Telemetry + module UI + # A QStandardItem to display and edit primitive ruby types in a tree view + class RubyItem < Vizkit::VizkitAccessorItem + def initialize + super(nil, :nil?) + setEditable false + end + + def setData(data, role = Qt::UserRole + 1) + return super if role != Qt::EditRole || data.isNull + + val = from_variant data, @getter.call + return false if val.nil? + return false unless val != @getter.call + + @setter.call val + modified! + end + + # Defines a block to be called to update the model when the item is edited + def setter(&block) + @setter = block + setEditable true + end + + # Defines a block to be called to fetch data from the model + def getter(&block) + @getter = block + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/runtime_state.rb b/lib/syskit/telemetry/ui/runtime_state.rb new file mode 100644 index 000000000..c5d26ebc5 --- /dev/null +++ b/lib/syskit/telemetry/ui/runtime_state.rb @@ -0,0 +1,665 @@ +# frozen_string_literal: true + +require "syskit" +require "roby/interface/async" +require "roby/interface/async/log" +require "syskit/telemetry/ui/logging_configuration" +require "syskit/telemetry/ui/job_status_display" +require "syskit/telemetry/ui/widget_list" +require "syskit/telemetry/ui/expanded_job_status" +require "syskit/telemetry/ui/global_state_label" +require "syskit/telemetry/ui/app_start_dialog" +require "syskit/telemetry/ui/batch_manager" + +module Syskit + module Telemetry + module UI + # UI that displays and allows to control jobs + class RuntimeState < Qt::Widget + include Roby::Hooks + include Roby::Hooks::InstanceHooks + + # @return [Roby::Interface::Async::Interface] the underlying syskit + # interface + attr_reader :syskit + # An async object to access the log stream + attr_reader :syskit_log_stream + + # The toplevel layout + attr_reader :main_layout + # The layout used to organize the widgets to create new jobs + attr_reader :new_job_layout + # The [WidgetList] widget in which we display the + # summary of job status + attr_reader :job_status_list + # The [ExpandedJobStatus] widget in which we display expanded job + # information + attr_reader :job_expanded_status + # The combo box used to create new jobs + attr_reader :action_combo + # The job that is currently selected + attr_reader :current_job + # The connection state, which gives access to the global Syskit + # state + attr_reader :connection_state + + # All known tasks + attr_reader :all_tasks + # Job information for tasks in the rebuilt plan + attr_reader :all_job_info + + # The name service which allows us to resolve Rock task contexts + attr_reader :name_service + # A task inspector widget we use to display the task states + attr_reader :ui_task_inspector + # A logging configuration widget we use to manage logging + attr_reader :ui_logging_configuration + # The list of task names of the task currently displayed by the task + # inspector + attr_reader :current_orocos_tasks + + # Returns a list of actions that can be performed on the Roby + # instance + # + # @return [Array] + attr_reader :global_actions + + # The current connection state + attr_reader :current_state + + # Checkboxes to select widgets options + attr_reader :ui_hide_loggers + attr_reader :ui_show_expanded_job + + define_hooks :on_connection_state_changed + define_hooks :on_progress + + class ActionListDelegate < Qt::StyledItemDelegate + OUTER_MARGIN = 5 + INTERLINE = 3 + def sizeHint(option, index) + fm = option.font_metrics + main = index.data.toString + doc = index.data(Qt::UserRole).to_string || "" + Qt::Size.new( + [fm.width(main), fm.width(doc)].max + 2 * OUTER_MARGIN, + fm.height * 2 + OUTER_MARGIN * 2 + INTERLINE + ) + end + + def paint(painter, option, index) + painter.save + + if (option.state & Qt::Style::State_Selected) != 0 + painter.fill_rect(option.rect, option.palette.highlight) + painter.brush = option.palette.highlighted_text + end + + main = index.data.toString + doc = index.data(Qt::UserRole).to_string || "" + text_bounds = Qt::Rect.new + + fm = option.font_metrics + painter.draw_text( + Qt::Rect.new(option.rect.x + OUTER_MARGIN, option.rect.y + OUTER_MARGIN, option.rect.width - 2 * OUTER_MARGIN, fm.height), + Qt::AlignLeft, main, text_bounds + ) + + font = painter.font + font.italic = true + painter.font = font + painter.draw_text( + Qt::Rect.new(option.rect.x + OUTER_MARGIN, text_bounds.bottom + INTERLINE, option.rect.width - 2 * OUTER_MARGIN, fm.height), + Qt::AlignLeft, doc, text_bounds + ) + ensure + painter.restore + end + end + + # @param [Roby::Interface::Async::Interface] syskit the underlying + # syskit interface + # @param [Integer] poll_period how often should the syskit interface + # be polled (milliseconds). Set to nil if the polling is already + # done externally + def initialize(parent: nil, robot_name: "default", + syskit: Roby::Interface::Async::Interface.new, poll_period: 50) + + super(parent) + + @syskit = syskit + @robot_name = robot_name + reset + + @syskit_poll = Qt::Timer.new + @syskit_poll_period = poll_period + connect syskit_poll, SIGNAL("timeout()"), + self, SLOT("poll_syskit_interface()") + + if poll_period + @syskit_poll.start(poll_period) + end + + create_ui + + @global_actions = {} + action = global_actions[:start] = Qt::Action.new("Start", self) + @starting_monitor = Qt::Timer.new + connect @starting_monitor, SIGNAL("timeout()"), + self, SLOT("monitor_syskit_startup()") + connect action, SIGNAL("triggered()") do + app_start(robot_name: @robot_name, port: syskit.remote_port) + end + action = global_actions[:restart] = Qt::Action.new("Restart", self) + connect action, SIGNAL("triggered()") do + app_restart + end + action = global_actions[:quit] = Qt::Action.new("Quit", self) + connect action, SIGNAL("triggered()") do + app_quit + end + + @current_job = nil + @current_orocos_tasks = Set.new + @all_tasks = Set.new + @known_loggers = nil + @all_job_info = {} + syskit.on_ui_event do |event_name, *args| + if w = @ui_event_widgets[event_name] + w.show + w.update(*args) + else + puts "don't know what to do with UI event #{event_name}, known events: #{@ui_event_widgets}" + end + end + on_connection_state_changed do |state| + @current_state = state + connection_state.update_state state + end + syskit.on_reachable do + @syskit_commands = syskit.client.syskit + update_log_server_connection(syskit.log_server_port) + @job_status_list.each_widget do |w| + w.show_actions = true + end + action_combo.clear + action_combo.enabled = true + syskit.actions.sort_by(&:name).each do |action| + next if action.advanced? + + action_combo.add_item(action.name, Qt::Variant.new(action.doc)) + end + ui_logging_configuration.refresh + global_actions[:start].visible = false + global_actions[:restart].visible = true + global_actions[:quit].visible = true + @starting_monitor.stop + run_hook :on_connection_state_changed, "CONNECTED" + end + syskit.on_unreachable do + @syskit_commands = nil + @job_status_list.each_widget do |w| + w.show_actions = false + end + @ui_event_widgets.each_value(&:hide) + action_combo.enabled = false + @batch_manager.cancel + if remote_name == "localhost" + global_actions[:start].visible = true + end + ui_logging_configuration.refresh + global_actions[:restart].visible = false + global_actions[:quit].visible = false + if @current_state != "RESTARTING" + run_hook :on_connection_state_changed, "UNREACHABLE" + end + end + syskit.on_job do |job| + job.start + monitor_job(job) + end + end + + def monitor_syskit_startup + return unless @syskit_pid + + begin + _pid, has_quit = Process.waitpid2( + @syskit_pid, Process::WNOHANG + ) + rescue Errno::ECHILD + has_quit = true + end + + if has_quit + @syskit_pid = nil + run_hook :on_connection_state_changed, "UNREACHABLE" + @starting_monitor.stop + end + end + slots "monitor_syskit_startup()" + + def reset + Orocos.initialize + @logger_m = nil + orocos_corba_nameservice = Orocos::CORBA::NameService.new(syskit.remote_name) + @name_service = Orocos::Async::NameService.new(orocos_corba_nameservice) + end + + def update_log_server_connection(port) + if syskit_log_stream && (syskit_log_stream.port == port) + return + elsif syskit_log_stream + syskit_log_stream.close + end + + @syskit_log_stream = Roby::Interface::Async::Log.new(syskit.remote_name, port: port) + syskit_log_stream.on_reachable do + deselect_job + end + syskit_log_stream.on_init_progress do |rx, expected| + run_hook :on_progress, format("loading %02i", Float(rx) / expected * 100) + end + syskit_log_stream.on_update do |cycle_index, cycle_time| + if syskit_log_stream.init_done? + time_s = cycle_time.strftime("%H:%M:%S.%3N").to_s + run_hook :on_progress, format("@%i %s", cycle_index, time_s) + + job_expanded_status.update_time(cycle_index, cycle_time) + update_tasks_info + job_expanded_status.add_tasks_info(all_tasks, all_job_info) + job_expanded_status.scheduler_state = syskit_log_stream.scheduler_state + job_expanded_status.update_chronicle unless hide_expanded_jobs? + end + syskit_log_stream.clear_integrated + end + end + + def hide_loggers? + !@ui_hide_loggers.checked? + end + + def hide_expanded_jobs? + !@ui_show_expanded_job.checked + end + + def remote_name + syskit.remote_name + end + + def app_start(robot_name: "default", port: nil) + robot_name, start_controller, single = AppStartDialog.exec( + Roby.app.robots.names, self, default_robot_name: robot_name + ) + return unless robot_name + + extra_args = [] + extra_args << "-r" << robot_name unless robot_name.empty? + extra_args << "-c" if start_controller + extra_args << "--port=#{port}" if port + extra_args << "--single" if single + extra_args.concat( + Roby.app.argv_set.flat_map { |arg| ["--set", arg] } + ) + @syskit_pid = + Kernel.spawn Gem.ruby, "-S", "syskit", "run", "--wait-shell-connection", + *extra_args, + pgroup: true + @starting_monitor.start(100) + run_hook :on_connection_state_changed, "STARTING" + end + + def app_quit + syskit.quit + end + + def app_restart + run_hook :on_connection_state_changed, "RESTARTING" + if @syskit_pid + @starting_monitor.start(100) + end + syskit.restart + end + + def logger_task?(t) + return if @logger_m == false + + @logger_m ||= Syskit::TaskContext + .find_model_from_orogen_name("logger::Logger") || false + t.kind_of?(@logger_m) if @logger_m + end + + def update_tasks_info + if current_job + job_task = syskit_log_stream.plan.find_tasks(Roby::Interface::Job) + .with_arguments(job_id: current_job.job_id) + .first + return unless job_task + + placeholder_task = job_task.planned_task || job_task + return unless placeholder_task + + dependency = placeholder_task.relation_graph_for(Roby::TaskStructure::Dependency) + tasks = dependency.enum_for(:depth_first_visit, placeholder_task).to_a + tasks << job_task + else + tasks = syskit_log_stream.plan.tasks + end + + if hide_loggers? + unless @known_loggers + @known_loggers = Set.new + all_tasks.delete_if do |t| + @known_loggers << t if logger_task?(t) + end + end + + tasks = tasks.find_all do |t| + if all_tasks.include?(t) + true + elsif @known_loggers.include?(t) + false + elsif logger_task?(t) + @known_loggers << t + false + else true + end + end + end + all_tasks.merge(tasks) + tasks.each do |job| + if job.kind_of?(Roby::Interface::Job) + placeholder_task = job.planned_task || job + all_job_info[placeholder_task] = job + end + end + update_orocos_tasks + end + + def update_orocos_tasks + candidate_tasks = all_tasks + .find_all { |t| t.kind_of?(Syskit::TaskContext) } + orocos_tasks = candidate_tasks.map { |t| t.arguments[:orocos_name] }.compact.to_set + removed = current_orocos_tasks - orocos_tasks + new = orocos_tasks - current_orocos_tasks + removed.each do |task_name| + ui_task_inspector.remove_task(task_name) + end + new.each do |task_name| + ui_task_inspector.add_task(name_service.proxy(task_name)) + end + @current_orocos_tasks = orocos_tasks + end + + EventWidget = Struct.new :name, :widget, :hook do + def show + widget.show + end + + def hide + widget.hide + end + + def update(*args) + hook.call(*args) + end + end + + def create_ui + job_summary = Qt::Widget.new + job_summary_layout = Qt::VBoxLayout.new(job_summary) + job_summary_layout.add_layout(@new_job_layout = create_ui_new_job) + + @connection_state = GlobalStateLabel.new(name: remote_name) + on_progress do |message| + state = connection_state.current_state.to_s + connection_state.update_text(format("%s - %s", state, message)) + end + job_summary_layout.add_widget(connection_state, 0) + + @clear_button = Qt::PushButton.new("Clear Finished Jobs") + job_summary_layout.add_widget(@clear_button) + @clear_button.connect(SIGNAL(:clicked)) do + @job_status_list.clear_widgets do |w| + unless w.job.active? + w.job.stop + true + end + end + end + + @batch_manager = BatchManager.new(@syskit, self) + job_summary_layout.add_widget(@batch_manager) + @batch_manager.connect(SIGNAL("active(bool)")) do |active| + if active then @batch_manager.show + else @batch_manager.hide + end + end + @batch_manager.hide + connection_state.connect(SIGNAL("clicked(QPoint)")) do + deselect_job + end + + @job_status_list = WidgetList.new(self) + @job_status_list.size_constraint = Qt::Layout::SetFixedSize + job_status_scroll = Qt::ScrollArea.new + job_status_scroll.widget = @job_status_list + job_summary_layout.add_widget(job_status_scroll, 1) + @main_layout = Qt::VBoxLayout.new(self) + + @ui_event_widgets = create_ui_event_widgets + @ui_event_widgets.each_value do |w| + @main_layout.add_widget(w.widget) + end + + splitter = Qt::Splitter.new + splitter.add_widget job_summary + splitter.add_widget(@job_expanded_status = ExpandedJobStatus.new) + connect(@job_expanded_status, SIGNAL("fileOpenClicked(const QUrl&)"), + self, SIGNAL("fileOpenClicked(const QUrl&)")) + + task_inspector_widget = Qt::Widget.new + task_inspector_layout = Qt::VBoxLayout.new(task_inspector_widget) + task_inspector_checkboxes_layout = Qt::HBoxLayout.new + task_inspector_checkboxes_layout.add_widget( + @ui_show_expanded_job = Qt::CheckBox.new("Show details") + ) + task_inspector_checkboxes_layout.add_widget( + @ui_hide_loggers = Qt::CheckBox.new("Show loggers") + ) + task_inspector_checkboxes_layout.add_stretch + task_inspector_layout.add_layout(task_inspector_checkboxes_layout) + task_inspector_layout.add_widget( + @ui_task_inspector = Vizkit.default_loader.TaskInspector + ) + @ui_hide_loggers.checked = false + @ui_hide_loggers.connect SIGNAL("toggled(bool)") do |checked| + @known_loggers = nil + update_tasks_info + end + @ui_show_expanded_job.checked = true + @ui_show_expanded_job.connect SIGNAL("toggled(bool)") do |checked| + job_expanded_status.visible = checked + end + + @ui_logging_configuration = LoggingConfiguration.new(syskit) + + management_tab_widget = Qt::TabWidget.new(self) + management_tab_widget.addTab(task_inspector_widget, "Tasks") + management_tab_widget.addTab(ui_logging_configuration, "Logging") + + splitter.add_widget(management_tab_widget) + job_expanded_status.set_size_policy(Qt::SizePolicy::MinimumExpanding, Qt::SizePolicy::MinimumExpanding) + @main_layout.add_widget splitter, 1 + w = splitter.size.width + splitter.sizes = [Integer(w * 0.25), Integer(w * 0.50), Integer(w * 0.25)] + nil + end + + def create_ui_event_frame + frame = Qt::Frame.new(self) + frame.frame_shape = Qt::Frame::StyledPanel + frame.setStyleSheet("QFrame { background-color: rgb(205,235,255); border-radius: 2px; }") + frame + end + + def create_ui_event_button(text) + button = Qt::PushButton.new(text) + button.flat = true + button + end + + def create_ui_event_orogen_config_changed + syskit_orogen_config_changed = create_ui_event_frame + layout = Qt::HBoxLayout.new(syskit_orogen_config_changed) + layout.add_widget(Qt::Label.new("oroGen configuration files changes on disk"), 1) + layout.add_widget(reload = create_ui_event_button("Reload")) + layout.add_widget(close = create_ui_event_button("Close")) + reload.connect(SIGNAL("clicked()")) do + @syskit_commands.async_reload_config do + syskit_orogen_config_changed.hide + end + end + close.connect(SIGNAL("clicked()")) do + syskit_orogen_config_changed.hide + end + EventWidget.new( + "syskit_orogen_config_changed", + syskit_orogen_config_changed, -> {} + ) + end + + def create_ui_event_orogen_config_reloaded + syskit_orogen_config_reloaded = create_ui_event_frame + layout = Qt::HBoxLayout.new(syskit_orogen_config_reloaded) + layout.add_widget(label = Qt::Label.new, 1) + layout.add_widget(apply = create_ui_event_button("Reconfigure")) + layout.add_widget(close = create_ui_event_button("Close")) + apply.connect(SIGNAL("clicked()")) do + @syskit_commands.async_redeploy do + syskit_orogen_config_reloaded.hide + end + end + close.connect(SIGNAL("clicked()")) do + syskit_orogen_config_reloaded.hide + end + syskit_orogen_config_reloaded_hook = lambda do |changed_tasks, changed_tasks_running| + if changed_tasks.empty? + label.text = "oroGen configuration updated" + apply.hide + elsif changed_tasks_running.empty? + label.text = "oroGen configuration modifications applied to #{changed_tasks.size} configured but not running tasks" + apply.hide + else + label.text = "oroGen configuration modifications applied to #{changed_tasks_running.size} running tasks and #{changed_tasks.size - changed_tasks_running.size} configured but not running tasks" + apply.show + end + end + EventWidget.new("syskit_orogen_config_reloaded", + syskit_orogen_config_reloaded, + syskit_orogen_config_reloaded_hook) + end + + def create_ui_event_widgets + widgets = [ + create_ui_event_orogen_config_reloaded, + create_ui_event_orogen_config_changed + ] + ui_event_widgets = {} + widgets.each do |w| + w.hide + ui_event_widgets[w.name] = w + end + ui_event_widgets + end + + def create_ui_new_job + new_job_layout = Qt::HBoxLayout.new + label = Qt::Label.new("New Job", self) + label.set_size_policy(Qt::SizePolicy::Minimum, Qt::SizePolicy::Minimum) + @action_combo = Qt::ComboBox.new(self) + action_combo.enabled = false + action_combo.item_delegate = ActionListDelegate.new(self) + new_job_layout.add_widget label + new_job_layout.add_widget action_combo, 1 + action_combo.connect(SIGNAL("activated(QString)")) do |action_name| + @batch_manager.create_new_job(action_name) + end + new_job_layout + end + + attr_reader :syskit_poll + + # @api private + # + # Sets up polling on a given syskit interface + def poll_syskit_interface + syskit.poll + if syskit_log_stream + if syskit_log_stream.poll(max: 0.05) == Roby::Interface::Async::Log::STATE_PENDING_DATA + syskit_poll.interval = 0 + else + syskit_poll.interval = @syskit_poll_period + end + end + end + slots "poll_syskit_interface()" + + # @api private + # + # Create the UI elements for the given job + # + # @param [Roby::Interface::Async::JobMonitor] job + def monitor_job(job) + job_status = JobStatusDisplay.new(job, @batch_manager) + job_status_list.add_widget job_status + job_status.connect(SIGNAL("clicked()")) do + select_job(job_status) + end + job_status.connect(SIGNAL("clearJob()")) do + job_status_list.clear_widgets do |job| + job == job_status + end + end + end + + def deselect_job + @current_job = nil + job_expanded_status.deselect + all_tasks.clear + @known_loggers = nil + all_job_info.clear + if syskit_log_stream + update_tasks_info + end + job_expanded_status.add_tasks_info(all_tasks, all_job_info) + end + + def select_job(job_status) + @current_job = job_status.job + all_tasks.clear + @known_loggers = nil + all_job_info.clear + update_tasks_info + job_expanded_status.select(job_status) + job_expanded_status.add_tasks_info(all_tasks, all_job_info) + end + + def restore_from_settings(settings) + %w{ui_hide_loggers ui_show_expanded_job}.each do |checkbox_name| + default = Qt::Variant.new(send(checkbox_name).checked) + send(checkbox_name).checked = settings.value(checkbox_name, default).to_bool + end + end + + def save_to_settings(settings) + %w(ui_hide_loggers ui_show_expanded_job).each do |checkbox_name| + settings.set_value checkbox_name, Qt::Variant.new(send(checkbox_name).checked) + end + end + + signals "fileOpenClicked(const QUrl&)" + end + end + end +end diff --git a/lib/syskit/telemetry/ui/state_label.rb b/lib/syskit/telemetry/ui/state_label.rb new file mode 100644 index 000000000..c9aa49df9 --- /dev/null +++ b/lib/syskit/telemetry/ui/state_label.rb @@ -0,0 +1,207 @@ +# frozen_string_literal: true + +module Syskit + module Telemetry + module UI + # Base class for the labels that represent an object and its states + class StateLabel < Qt::Label + COLORS = Hash[ + blue: "rgb(51, 181, 229)", + green: "rgb(153, 204, 0)", + red: "rgb(255, 68, 68)"] + + STYLE = "QLabel { padding: 3; background-color: %s; %s }" + TEXT_WITH_NAME = "%s: %s" + TEXT_WITHOUT_NAME = "%s" + + # The name that should be displayed in addition to the state + # + # If left to nil (the default in {#initialize}), no name will be + # displayed at all + # + # @return [nil,String] + attr_reader :name + + # Sets or resets the name that should be displayed in addition to + # the state + # + # Set to nil to remove any additional text + def name=(name) + @name = name + update_text + end + + # The current state + # + # @return [String] + attr_reader :current_state + + # The current text + # + # @return [String] + attr_reader :current_text + + # The current color + # + # @return [String] + attr_reader :current_color + + # Set of known states + # + # StateLabel defines 'INIT' and binds it to the blue color + attr_reader :states + + # Extra styling elements that should be added to the label + # stylesheet + attr_reader :extra_style + + # Sets {#extra_style} + def extra_style=(style) + @extra_style = style.to_str + update_style + end + + # The default color that will be used for undeclared states + # + # If nil, calling {#update_state} with an unknown state will raise + # an exception + attr_reader :default_color + + # Sets {#default_color} + def default_color=(color) + @default_color = handle_color_argument(color) + end + + def initialize(name: nil, extra_style: "", parent: nil, rate_limited: false) + super(parent) + @rate_limited = rate_limited + @name = name + @extra_style = extra_style + @states = {} + + declare_state :INIT, :blue + update_state :INIT + end + + # Declare that the given state should be ignored + # + # The display will not be changed when the state changes to an + # ignored state + # + # @param [String] state_name the name of the state that should be + # ignored + def ignore_state(state_name) + states[state_name.to_s] = nil + end + + # @api private + # + # Helper to handle a color argument + # + # @param [String] color the color name (in {COLORS}) or a + # stylesheet color (e.g. rgb(20, 30, 50)). Anything that is not a + # key in {COLOR} is interpreted as a stylesheet color + # @return [String] a stylesheet color + def handle_color_argument(color) + if c = COLORS[color] + COLORS[color] + else + color.to_str + end + end + + # Associate a state name and a color + # + # @param [String] state_name the state name + # @param [String] color the color. It can either be a color name in + # {COLOR} or a Qt stylesheet color (e.g. 'rgb(20, 30, 50)'). Any + # string that is not a color name will be interpreted as a + # stylesheet color (i.e. no validation is made) + def declare_state(state_name, color) + states[state_name.to_s] = handle_color_argument(color) + self + end + + # Declare a color for non-declared states + # + # If unset (the default), a non-declared state will be interpreted + # as an error. Otherwise, this color will be chosen + def declare_default_color(color) + self.default_color = handle_color_argument(color) + self + end + + # Returns the color that should be used for a given state + # + # @param [String] state the state name + # @return [String] the Qt stylesheet color as defined with + # {#declare_state} or, if the state has not been declared, by + # {#declare_default_color} + # + # @raise [ArgumentError] if the state has not been declared with + # {#declare_state} and no defeault color has been set with + # {#declare_default_color} + def color_from_state(state) + state = state.to_s + if states.key?(state) + states[state] + elsif color = default_color + color + else + raise ArgumentError, "unknown state #{state} and no default color defined" + end + end + + # Update to reflect a state change + # + # @param [String] state the state name + # @param [String] text the text to be displayed for this state + # change + # @param [String] color the color to use for this state + def update_state(state, text: state.to_s, color: color_from_state(state)) + return unless color + + update_style(color) + update_text(text) + @current_state = state.to_s + end + + # Update the label's style to use the given color + # + # @param [String] color a Qt stylesheet color (e.g. rgb(20,30,50)) + def update_style(color = current_color) + @current_color = color + color = handle_color_argument(color) + self.style_sheet = format(STYLE, color, extra_style) + end + + def rate_limited? + @rate_limited + end + + # Update the displayed text + # + # If {#name} is set, the resulting text is name: text, otherwise + # just text + # + # The text is displayed using the {#current_color} and + # {#extra_style} + def update_text(text = current_text) + above_rate_limit = @last_update && (Time.now - @last_update) < 1 + return if rate_limited? && above_rate_limit + + @last_update = Time.now + + text = text.to_str + @current_text = text + self.text = + if name then format(TEXT_WITH_NAME, name, text) + else format(TEXT_WITHOUT_NAME, text) + end + end + + slots "update_state(QString)" + end + end + end +end diff --git a/lib/syskit/telemetry/ui/widget_list.rb b/lib/syskit/telemetry/ui/widget_list.rb new file mode 100644 index 000000000..e4db27a21 --- /dev/null +++ b/lib/syskit/telemetry/ui/widget_list.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +module Syskit + module Telemetry + module UI + class WidgetList < Qt::Widget + def auto_resize? + @auto_resize + end + + ListItem = Struct.new :widget, :permanent do + def job + widget.job + end + + def permanent? + permanent + end + end + + def initialize(parent = nil, auto_resize: true) + super(parent) + + @main_layout = Qt::VBoxLayout.new(self) + @widgets = [] + self.size_constraint = Qt::Layout::SetMinAndMaxSize + @auto_resize = auto_resize + if auto_resize + @main_layout.add_stretch(1) + end + @separators = {} + end + + def size_constraint=(constraint) + @main_layout.size_constraint = constraint + end + + def add_separator(name, label, permanent: true) + add_widget(w = Qt::Label.new(label, self), permanent: permanent) + @separators[name] = w + w + end + + def add_before(widget, before, permanent: false) + if before.respond_to?(:to_str) + before = @separators.fetch(before) + end + + if i = @widgets.index { |w| w.widget == before } + @main_layout.insert_widget(i, widget) + @widgets.insert(i, ListItem.new(widget, permanent)) + else Kernel.raise ArgumentError, "#{before} is not part of #{self}" + end + end + + def add_after(widget, after, permanent: false) + if after.respond_to?(:to_str) + after = @separators.fetch(after) + end + + if i = @widgets.index { |w| w.widget == after } + @main_layout.insert_widget(i + 1, widget) + @widgets.insert(i + 1, ListItem.new(widget, permanent)) + else Kernel.raise ArgumentError, "#{after} is not part of #{self}" + end + end + + def add_widget(w, permanent: false) + @widgets << ListItem.new(w, permanent) + if auto_resize? + @main_layout.insert_widget(@widgets.size - 1, w) + else + @main_layout.add_widget(w) + end + end + + def children_size_updated + s = size + new_height = @widgets.inject(0) do |h, w| + h + if w.widget.hidden? then 0 + else w.widget.contents_height + end + end + if new_height != s.height + self.size = s + end + end + + def resizeEvent(event) + s = size + s.width = event.size.width + self.size = s + event.accept + end + + # Enumerate the widgets in the list + def each_widget + return enum_for(__method__) unless block_given? + + @widgets.each do |item| + yield(item.widget) + end + end + + # Clear widgets for which the given filter returns true + # + # It removes all widgets if no filter is given + # + # @yieldparam [ListItem] item + def clear_widgets(&filter) + filter ||= ->(w) { true } + separators = @separators.values + + kept_widgets = [] + until @widgets.empty? + w = @widgets.last + if !separators.include?(w.widget) && !w.permanent? && filter[w.widget] + @main_layout.remove_widget(w.widget) + w.widget.dispose + else + kept_widgets.unshift w + end + @widgets.pop + end + ensure + @widgets.concat(kept_widgets) + end + end + end + end +end From 72de283faa6dfbdba4a92dbeb0262819b24655d1 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 4 Jul 2023 11:05:43 -0300 Subject: [PATCH 152/260] chore: move syskit/shell_interface into syskit/interface/commands To prepare for new parts for the interface. --- .rubocop.yml | 2 +- .rubocop_todo.yml | 12 -- lib/syskit/gui/logging_configuration.rb | 2 +- lib/syskit/interface.rb | 3 + lib/syskit/interface/commands.rb | 253 +++++++++++++++++++++++ lib/syskit/roby_app/register_plugin.rb | 2 +- lib/syskit/shell_interface.rb | 216 ------------------- test/interface/test_commands.rb | 263 ++++++++++++++++++++++++ test/test_shell_interface.rb | 249 ---------------------- 9 files changed, 522 insertions(+), 480 deletions(-) create mode 100644 lib/syskit/interface.rb create mode 100644 lib/syskit/interface/commands.rb delete mode 100644 lib/syskit/shell_interface.rb create mode 100644 test/interface/test_commands.rb delete mode 100644 test/test_shell_interface.rb diff --git a/.rubocop.yml b/.rubocop.yml index 50471bf76..20adaae4d 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -47,4 +47,4 @@ Security/MarshalLoad: Style/MultilineBlockChain: Exclude: - - test/**/test_*.rb \ No newline at end of file + - test/**/test_*.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index b5f6e3277..6a78ea17f 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -99,7 +99,6 @@ Layout/LineLength: - 'lib/syskit/scripts/ide.rb' - 'lib/syskit/scripts/instanciate.rb' - 'lib/syskit/scripts/instanciate_gui.rb' - - 'lib/syskit/shell_interface.rb' - 'lib/syskit/task_configuration_manager.rb' - 'lib/syskit/test/base.rb' - 'lib/syskit/test/profile_test.rb' @@ -172,7 +171,6 @@ Layout/LineLength: - 'test/test_properties.rb' - 'test/test_property.rb' - 'test/test_remote_state_getter.rb' - - 'test/test_shell_interface.rb' - 'test/test_task_configuration_manager.rb' - 'test/test_task_context.rb' @@ -212,7 +210,6 @@ Lint/AmbiguousRegexpLiteral: - 'test/test_component.rb' - 'test/test_data_service.rb' - 'test/test_remote_state_getter.rb' - - 'test/test_shell_interface.rb' # Offense count: 114 # Configuration parameters: AllowSafeAssignment. @@ -329,7 +326,6 @@ Lint/UnusedBlockArgument: - 'lib/syskit/roby_app/plugin.rb' - 'lib/syskit/ros/node.rb' - 'lib/syskit/runtime/connection_management.rb' - - 'lib/syskit/shell_interface.rb' - 'test/coordination/test_data_monitoring_table.rb' - 'test/coordination/test_models_task_extension.rb' - 'test/models/test_specialization_manager.rb' @@ -495,7 +491,6 @@ Metrics/AbcSize: - 'lib/syskit/runtime/update_task_states.rb' - 'lib/syskit/scripts/common.rb' - 'lib/syskit/scripts/doc.rb' - - 'lib/syskit/shell_interface.rb' - 'lib/syskit/task_context.rb' - 'lib/syskit/test/network_manipulation.rb' - 'lib/syskit/test/profile_assertions.rb' @@ -738,7 +733,6 @@ Naming/ClassAndModuleCamelCase: Naming/MemoizedInstanceVariableName: Exclude: - 'lib/syskit/roby_app/rest_api.rb' - - 'lib/syskit/shell_interface.rb' # Offense count: 21 # Configuration parameters: EnforcedStyle, IgnoredPatterns. @@ -903,7 +897,6 @@ Style/Documentation: - 'lib/syskit/scripts/common.rb' - 'lib/syskit/scripts/connection_log.rb' - 'lib/syskit/scripts/doc.rb' - - 'lib/syskit/shell_interface.rb' - 'lib/syskit/test.rb' - 'lib/syskit/test/base.rb' - 'lib/syskit/test/component_test.rb' @@ -1039,7 +1032,6 @@ Style/HashSyntax: - 'lib/syskit/scripts/autodetect_interfaces.rb' - 'lib/syskit/scripts/common.rb' - 'lib/syskit/scripts/doc.rb' - - 'lib/syskit/shell_interface.rb' - 'test/models/test_composition_specialization.rb' - 'test/models/test_deployment_group.rb' - 'test/models/test_specialization_manager.rb' @@ -1108,7 +1100,6 @@ Style/IfUnlessModifier: - 'lib/syskit/scripts/browse.rb' - 'lib/syskit/scripts/common.rb' - 'lib/syskit/scripts/instanciate.rb' - - 'lib/syskit/shell_interface.rb' - 'lib/syskit/task_configuration_manager.rb' - 'lib/syskit/test/base.rb' - 'lib/syskit/test/flexmock_extension.rb' @@ -1118,7 +1109,6 @@ Style/IfUnlessModifier: - 'test/runtime/test_connection_management.rb' - 'test/test_deployment.rb' - 'test/test_remote_state_getter.rb' - - 'test/test_shell_interface.rb' - 'test/test_task_context.rb' # Offense count: 3 @@ -1142,7 +1132,6 @@ Style/MultilineBlockChain: - 'test/network_generation/test_engine.rb' - 'test/runtime/test_connection_management.rb' - 'test/test_deployment.rb' - - 'test/test_shell_interface.rb' # Offense count: 15 # Cop supports --auto-correct. @@ -1170,7 +1159,6 @@ Style/RedundantBegin: - 'lib/syskit/port.rb' - 'lib/syskit/roby_app/rest_api.rb' - 'lib/syskit/runtime/connection_management.rb' - - 'lib/syskit/shell_interface.rb' - 'test/coordination/test_task_script.rb' - 'test/models/test_composition.rb' - 'test/test/test_task_context_test.rb' diff --git a/lib/syskit/gui/logging_configuration.rb b/lib/syskit/gui/logging_configuration.rb index 4642121a8..4b9d0d452 100644 --- a/lib/syskit/gui/logging_configuration.rb +++ b/lib/syskit/gui/logging_configuration.rb @@ -4,7 +4,7 @@ require "vizkit/vizkit_items" require "vizkit/tree_view" require "Qt4" -require "syskit/shell_interface" +require "syskit/interface" require "syskit/gui/logging_configuration_item" require "roby/interface/exceptions" diff --git a/lib/syskit/interface.rb b/lib/syskit/interface.rb new file mode 100644 index 000000000..ef6098227 --- /dev/null +++ b/lib/syskit/interface.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require "syskit/interface/commands" \ No newline at end of file diff --git a/lib/syskit/interface/commands.rb b/lib/syskit/interface/commands.rb new file mode 100644 index 000000000..547f6a3e2 --- /dev/null +++ b/lib/syskit/interface/commands.rb @@ -0,0 +1,253 @@ +# frozen_string_literal: true + +require "roby/interface/core" +require "roby/robot" + +module Syskit + module Interface + # Definition of the syskit-specific interface commands + class Commands < Roby::Interface::CommandLibrary + # Save the configuration of all running tasks of the given model to disk + # + # @param [String,nil] name the section name for the new configuration. + # If nil, the task's orocos name will be used + # @param [String] path the directory in which the files should be saved + # @return [nil] + def dump_task_config(task_model, path, name = nil) + FileUtils.mkdir_p(path) + plan.find_tasks(task_model) + .each do |t| + Orocos.conf.save(t.orocos_task, path, name || t.orocos_task.name) + end + nil + end + command( + :dump_task_config, + "saves configuration from running tasks into yaml files", + model: "the model of the tasks that should be saved", + path: "the directory in which the configuration files should be saved", + name: "(optional) if given, the name of the section for the new "\ + "configuration. Defaults to the orocos task names" + ) + + # Saves the configuration of all running tasks to disk + # + # @param [String] name the section name for the new configuration + # @param [String] path the directory in which the files should be saved + # @return [nil] + def dump_all_config(path, name = nil) + dump_task_config(Syskit::TaskContext, path, name) + nil + end + command( + :dump_all_config, + "saves the configuration of all running tasks into yaml files", + path: "the directory in which the configuration files should be saved", + name: "(optional) if given, the name of the section for the new "\ + "configuration. Defaults to the orocos task names" + ) + + # Task used to tell the engine that we want the deployments restarted without + # killing the dependent networks + class ShellDeploymentRestart < Roby::Task + event :start, controlable: true + + poll do + if redeploy_event.pending? && !plan.syskit_has_async_resolution? + redeploy_event.emit + end + end + + event :redeploy do |_context| + Runtime.apply_requirement_modifications(plan, force: true) + end + + forward redeploy: :stop + end + + # Stops deployment processes + # + # @param [Array>] models if non-empty, only the + # deployments matching this model will be stopped, otherwise all + # deployments are stopped. + def stop_deployments(*models) + models << Syskit::Deployment if models.empty? + models.each do |m| + plan.find_tasks(m) + .each do |task| + if task.kind_of?(Syskit::TaskContext) + task.execution_agent.stop! + else + task.stop! + end + end + end + end + command :stop_deployments, "stops deployment processes", + models: "(optional) if given, a list of task or deployment models "\ + "pointing to what should be stopped. If not given, all "\ + "deployments are stopped" + + # Restarts deployment processes + # + # @param [Array,Model>] models if + # non-empty, only the deployments matching this model or the deployments + # supporting tasks matching the models will be restarted, otherwise all + # deployments are restarted. + def restart_deployments(*models) + models << Syskit::Deployment if models.empty? + deployments = restart_discover_deployment_tasks(models) + protection = restart_setup_protection(deployments) + + done = Roby::AndGenerator.new + done.signals protection.redeploy_event + deployments.each do |task| + done << task.stop_event + task.stop! + end + nil + end + command :restart_deployments, "restarts deployment processes", + models: "(optional) if given, a list of task or deployment models "\ + "pointing to what should be restarted. If not given, all "\ + "deployments are restarted" + + # @api private + # + # Helper for {#restart_deployments} that lists the deployment tasks which + # should be restarted + def restart_discover_deployment_tasks(models) + tasks = models.flat_map do |m| + plan.find_tasks(m).map do |task| + if task.kind_of?(Syskit::TaskContext) + task.execution_agent + else + task + end + end + end + tasks.uniq + end + + # @api private + # + # Helper for {#restart_deployments} that sets up a error handler to avoid + # killing tasks while we restart the deployments + def restart_setup_protection(deployment_tasks) + protection = ShellDeploymentRestart.new + plan.add(protection) + protection.start! + + deployment_tasks.each do |task| + task.each_executed_task do |executed_task| + executed_task.stop_event.handle_with(protection) + end + end + protection + end + + # (see Application#syskit_reload_config) + def reload_config + app.syskit_reload_config + end + command :reload_config, "reloads YAML configuration files from disk", + "You need to call the redeploy command to apply the new configuration" + + # (see Application#syskit_pending_reloaded_configurations) + def pending_reloaded_configurations + app.syskit_pending_reloaded_configurations + end + command :pending_reloaded_configurations, + "returns the list of TaskContext names "\ + "that are marked as needing reconfiguration", + "They will be reconfigured on the next redeploy or system transition" + + # Require the engine to redeploy the current network + # + # It must be called after {#reload_config} to apply the new + # configuration(s) + def redeploy + Runtime.apply_requirement_modifications(plan, force: true) + nil + end + command :redeploy, "redeploys the current network", + "It is mostly used to apply the configuration "\ + "loaded with reload_config" + + def enable_log_group(string) + Syskit.conf.logs.enable_log_group(string) + redeploy + nil + end + command :enable_logging_of, "enables a log group", + name: "the log group name" + + # @deprecated use enable_log_group instead + def enable_logging_of(string) + enable_log_group(string) + end + + def disable_log_group(string) + Syskit.conf.logs.disable_log_group(string) + redeploy + nil + end + command :disable_log_group, "disables a log group", + name: "the log group name" + + # @deprecated use disable_log_group instead + def disable_logging_of(string) + disable_log_group(string) + end + + LoggingGroup = Struct.new(:name, :enabled) + LoggingConfiguration = + Struct.new(:port_logs_enabled, :conf_logs_enabled, :groups) + def logging_conf + conf = LoggingConfiguration.new(false, false, {}) + conf.port_logs_enabled = Syskit.conf.logs.port_logs_enabled? + conf.conf_logs_enabled = Syskit.conf.logs.conf_logs_enabled? + Syskit.conf.logs.groups.each_pair do |key, group| + conf.groups[key] = LoggingGroup.new(key, group.enabled?) + end + conf + end + command :logging_conf, "gets the current logging configuration" + + def update_logging_conf(conf) + logs_conf = Syskit.conf.logs + if conf.port_logs_enabled + logs_conf.enable_port_logging + else + logs_conf.disable_port_logging + end + + if conf.conf_logs_enabled + logs_conf.enable_conf_logging + else + logs_conf.disable_conf_logging + end + + conf.groups.each_pair do |name, group| + logs_conf.group_by_name(name).enabled = group.enabled + rescue ArgumentError + Syskit.warn "tried to update a group that does not exist: #{name}" + end + redeploy + end + command :update_logging_conf, "updates the current logging configuration", + conf: "the new logging settings" + end + end +end + +module Robot # :nodoc: + # Syskit subcommand for the shell + def self.syskit + @syskit_interface ||= Syskit::Interface::Commands.new(Roby.app) # rubocop:disable Naming/MemoizedInstanceVariableName + end +end + +Roby::Interface::Interface.subcommand( + "syskit", Syskit::Interface::Commands, "Commands specific to Syskit" +) diff --git a/lib/syskit/roby_app/register_plugin.rb b/lib/syskit/roby_app/register_plugin.rb index f0a7f6cc2..bcf04c3f0 100644 --- a/lib/syskit/roby_app/register_plugin.rb +++ b/lib/syskit/roby_app/register_plugin.rb @@ -2,6 +2,6 @@ require "syskit" Roby::Application.register_plugin("syskit", Syskit::RobyApp::Plugin) do - require "syskit/shell_interface" + require "syskit/interface" Syskit::RobyApp::Plugin.enable end diff --git a/lib/syskit/shell_interface.rb b/lib/syskit/shell_interface.rb deleted file mode 100644 index adbecef91..000000000 --- a/lib/syskit/shell_interface.rb +++ /dev/null @@ -1,216 +0,0 @@ -# frozen_string_literal: true - -require "roby/interface" -require "roby/robot" -module Syskit - # Definition of the syskit-specific interface commands - class ShellInterface < Roby::Interface::CommandLibrary - # Save the configuration of all running tasks of the given model to disk - # - # @param [String,nil] name the section name for the new configuration. - # If nil, the task's orocos name will be used - # @param [String] path the directory in which the files should be saved - # @return [nil] - def dump_task_config(task_model, path, name = nil) - FileUtils.mkdir_p(path) - plan.find_tasks(task_model) - .each do |t| - Orocos.conf.save(t.orocos_task, path, name || t.orocos_task.name) - end - nil - end - command :dump_task_config, "saves configuration from running tasks into yaml files", - :model => "the model of the tasks that should be saved", - :path => "the directory in which the configuration files should be saved", - :name => "(optional) if given, the name of the section for the new configuration. Defaults to the orocos task names" - - # Saves the configuration of all running tasks to disk - # - # @param [String] name the section name for the new configuration - # @param [String] path the directory in which the files should be saved - # @return [nil] - def dump_all_config(path, name = nil) - dump_task_config(Syskit::TaskContext, path, name) - nil - end - command :dump_all_config, "saves the configuration of all running tasks into yaml files", - :path => "the directory in which the configuration files should be saved", - :name => "(optional) if given, the name of the section for the new configuration. Defaults to the orocos task names" - - class ShellDeploymentRestart < Roby::Task - event :start, :controlable => true - - poll do - if redeploy_event.pending? && !plan.syskit_has_async_resolution? - redeploy_event.emit - end - end - - event :redeploy do |context| - Runtime.apply_requirement_modifications(plan, force: true) - end - - forward :redeploy => :stop - end - - # Stops deployment processes - # - # @param [Array>] models if non-empty, only the - # deployments matching this model will be stopped, otherwise all - # deployments are stopped. - def stop_deployments(*models) - if models.empty? - models << Syskit::Deployment - end - models.each do |m| - plan.find_tasks(m) - .each do |task| - if task.kind_of?(Syskit::TaskContext) - task.execution_agent.stop! - else - task.stop! - end - end - end - end - command :stop_deployments, "stops deployment processes", - :models => "(optional) if given, a list of task or deployment models pointing to what should be stopped. If not given, all deployments are stopped" - - # Restarts deployment processes - # - # @param [Array,Model>] models if - # non-empty, only the deployments matching this model or the deployments - # supporting tasks matching the models will be restarted, otherwise all - # deployments are restarted. - def restart_deployments(*models) - protection = ShellDeploymentRestart.new - plan.add(protection) - protection.start! - - if models.empty? - models << Syskit::Deployment - end - done = Roby::AndGenerator.new - done.signals protection.redeploy_event - - models.each do |m| - agents = plan.find_tasks(m).each_with_object(Set.new) do |task, result| - result << - if task.kind_of?(Syskit::TaskContext) - task.execution_agent - else - task - end - end - - agents.each do |agent_task| - agent_task.each_executed_task do |task| - task.stop_event.handle_with(protection) - end - done << agent_task.stop_event - agent_task.stop! - end - end - nil - end - command :restart_deployments, "restarts deployment processes", - :models => "(optional) if given, a list of task or deployment models pointing to what should be restarted. If not given, all deployments are restarted" - - # (see Application#syskit_reload_config) - def reload_config - app.syskit_reload_config - end - command :reload_config, "reloads YAML configuration files from disk", - "You need to call the redeploy command to apply the new configuration" - - # (see Application#syskit_pending_reloaded_configurations) - def pending_reloaded_configurations - app.syskit_pending_reloaded_configurations - end - command :pending_reloaded_configurations, "returns the list of TaskContext names that are marked as needing reconfiguration", - "They will be reconfigured on the next redeploy or system transition" - - # Require the engine to redeploy the current network - # - # It must be called after {#reload_config} to apply the new - # configuration(s) - def redeploy - Runtime.apply_requirement_modifications(plan, force: true) - nil - end - command :redeploy, "redeploys the current network", - "It is mostly used to apply the configuration loaded with reload_config" - - def enable_log_group(string) - Syskit.conf.logs.enable_log_group(string) - redeploy - nil - end - command :enable_logging_of, "enables a log group", - name: "the log group name" - - # @deprecated use enable_log_group instead - def enable_logging_of(string) - enable_log_group(string) - end - - def disable_log_group(string) - Syskit.conf.logs.disable_log_group(string) - redeploy - nil - end - command :disable_log_group, "disables a log group", - name: "the log group name" - - # @deprecated use disable_log_group instead - def disable_logging_of(string) - disable_log_group(string) - end - - LoggingGroup = Struct.new(:name, :enabled) - LoggingConfiguration = Struct.new(:port_logs_enabled, :conf_logs_enabled, :groups) - def logging_conf - conf = LoggingConfiguration.new(false, false, {}) - conf.port_logs_enabled = Syskit.conf.logs.port_logs_enabled? - conf.conf_logs_enabled = Syskit.conf.logs.conf_logs_enabled? - Syskit.conf.logs.groups.each_pair do |key, group| - conf.groups[key] = LoggingGroup.new(key, group.enabled?) - end - conf - end - command :logging_conf, "gets the current logging configuration" - - def update_logging_conf(conf) - if conf.port_logs_enabled - Syskit.conf.logs.enable_port_logging - else - Syskit.conf.logs.disable_port_logging - end - - if conf.conf_logs_enabled - Syskit.conf.logs.enable_conf_logging - else - Syskit.conf.logs.disable_conf_logging - end - - conf.groups.each_pair do |name, group| - begin - Syskit.conf.logs.group_by_name(name).enabled = group.enabled - rescue ArgumentError - Syskit.warn "tried to update a group that does not exist: #{name}" - end - end - redeploy - end - command :update_logging_conf, "updates the current logging configuration", - conf: "the new logging settings" - end -end - -module Robot - def self.syskit - @syskit_interface ||= Syskit::ShellInterface.new(Roby.app) - end -end - -Roby::Interface::Interface.subcommand "syskit", Syskit::ShellInterface, "Commands specific to Syskit" diff --git a/test/interface/test_commands.rb b/test/interface/test_commands.rb new file mode 100644 index 000000000..df5aa119a --- /dev/null +++ b/test/interface/test_commands.rb @@ -0,0 +1,263 @@ +# frozen_string_literal: true + +require "syskit/test/self" +require "syskit/interface" + +module Syskit + module Interface + describe Commands do + attr_reader :subject + + before do + @subject = Commands.new(flexmock(plan: plan)) + subject.execution_engine.thread = Thread.current + @interface_thread = nil + end + + after do + @ee_thread&.join + end + + describe "#redeploy" do + it "triggers a full deployment" do + flexmock(Runtime).should_receive(:apply_requirement_modifications) + .with(subject.plan, force: true).once.pass_thru + subject.redeploy + end + end + + describe "#restart_deployments" do + attr_reader :task_m, :task + before do + @task_m = TaskContext.new_submodel + @task = syskit_stub_deploy_configure_and_start( + syskit_stub_requirements(task_m).with_conf("default") + ) + plan.add_mission_task(task) + end + + it "stops the matching deployments and redeploys" do + plug_apply_requirement_modifications + expect_execution do + subject.restart_deployments + end.to do + emit find_tasks(Commands::ShellDeploymentRestart).stop_event + end + assert_equal 1, plan.find_tasks(task_m).pending.to_a.size + end + + it "restricts the deployments to the given models" do + other = syskit_stub_deploy_configure_and_start( + syskit_stub_requirements(TaskContext.new_submodel) + ) + plug_apply_requirement_modifications + expect_execution do + subject.restart_deployments(task.execution_agent.model) + end.to do + emit find_tasks(Commands::ShellDeploymentRestart).stop_event + emit task.stop_event + not_emit other.stop_event + end + assert_equal 1, plan.find_tasks(task_m).pending.to_a.size + end + + it "accepts task models as argument" do + other = syskit_stub_deploy_configure_and_start( + syskit_stub_requirements(TaskContext.new_submodel) + ) + plug_apply_requirement_modifications + expect_execution do + subject.restart_deployments(task.model) + end.to do + emit find_tasks(Commands::ShellDeploymentRestart).stop_event + emit task.stop_event + not_emit other.stop_event + end + assert_equal 1, plan.find_tasks(task_m).pending.to_a.size + end + end + + describe "#stop_deployments" do + attr_reader :task_m, :task + before do + @task_m = TaskContext.new_submodel + @task = syskit_stub_deploy_configure_and_start( + task_m.with_conf("default") + ) + plan.add_mission_task(task) + end + + it "stops the matching deployments" do + expect_execution { subject.stop_deployments } + .to do + emit task.aborted_event + emit task.execution_agent.stop_event + end + assert task.finished? + end + + it "restricts the deployments to the given models" do + other = + syskit_stub_deploy_configure_and_start(task_m.with_conf("other")) + subject.plan.add_mission_task(other) + expect_execution do + subject.stop_deployments(task.execution_agent.model) + end.to do + emit task.aborted_event + emit task.execution_agent.stop_event + end + assert task.finished? + assert !other.finished? + end + + it "accepts task models as argument" do + other_m = TaskContext.new_submodel + other = syskit_stub_deploy_configure_and_start(other_m) + subject.plan.add_mission_task(other) + expect_execution do + subject.stop_deployments(task.execution_agent.model) + end.to do + emit task.aborted_event + emit task.execution_agent.stop_event + end + assert task.finished? + assert !other.finished? + end + end + + describe "the log configuration management" do + before { Syskit.conf.logs.create_group "test" } + after { Syskit.conf.logs.remove_group("test") } + + it "creates a marshallable instance of the configuration" do + conf = subject.logging_conf + assert_equal conf.port_logs_enabled, + Syskit.conf.logs.port_logs_enabled? + assert_equal conf.conf_logs_enabled, + Syskit.conf.logs.conf_logs_enabled? + Syskit.conf.logs.groups.each_pair do |key, group| + assert_equal group.enabled?, conf.groups[key].enabled + end + Marshal.dump(conf) + end + + it "changes status of conf and port logging and redeploys" do + conf = subject.logging_conf + previous_port_status = Syskit.conf.logs.port_logs_enabled? + previous_conf_status = Syskit.conf.logs.conf_logs_enabled? + + conf.port_logs_enabled = !previous_port_status + conf.conf_logs_enabled = !previous_conf_status + + flexmock(subject).should_receive(:redeploy).once.pass_thru do + assert_equal Syskit.conf.logs.port_logs_enabled?, + !previous_port_status + assert_equal Syskit.conf.logs.conf_logs_enabled?, + !previous_conf_status + end + subject.update_logging_conf(conf) + end + + it "changes status of an existing log group and redeploys" do + conf = subject.logging_conf + previous_status = Syskit.conf.logs.group_by_name("test").enabled? + conf.groups["test"].enabled = !previous_status + + flexmock(subject).should_receive(:redeploy).once.pass_thru do + assert_equal Syskit.conf.logs.group_by_name("test").enabled?, + !previous_status + end + subject.update_logging_conf(conf) + end + end + + describe "the log group management" do + attr_reader :group + before do + @group = Syskit.conf.logs.create_group "test" do |g| + g.add(/base.samples.frame.Frame/) + end + end + + after do + Syskit.conf.logs.remove_group("test") + end + + it "enable_log_group enables the log group and redeploys" do + group.enabled = false + flexmock(subject).should_receive(:redeploy).once.ordered + subject.enable_log_group "test" + assert group.enabled? + end + + it "disable_log_group enables the log group and redeploys" do + group.enabled = true + flexmock(subject).should_receive(:redeploy).once.ordered + subject.disable_log_group "test" + assert !group.enabled? + end + + it "enable_log_group raises ArgumentError "\ + "if the log group does not exist" do + assert_raises(ArgumentError) do + subject.enable_log_group "does_not_exist" + end + end + + it "disable_log_group raises ArgumentError "\ + "if the log group does not exist" do + assert_raises(ArgumentError) do + subject.disable_log_group "does_not_exist" + end + end + end + + # Start a thread to do a call on the interface that is synchronized + # with {ExecutionEngine#execute} + # + # @example call the 'redeploy' interface command, and wait for its + # result + # + # # Call 'redeploy'. The queue method waits 50ms to give time to + # # the thread to actually start the call. There's no way to be + # # sure, so that might lead to random test failures + # queue_execute_call { subject.redeploy } + # # Force processing of the call, and wait for the thread to + # # finish + # process_execute_call + def queue_execute_call(&block) + if @interface_thread + raise "you must call #process_execute_call after a call "\ + "to #queue_execute_call" + end + + @interface_thread_sync = sync = Concurrent::CyclicBarrier.new(2) + @interface_thread = Thread.new do + sync.wait + block.call + sync.wait + end + sync.wait + sleep 0.05 + end + + # Process the work queued with {#queue_execute_call} + def process_execute_call + subject.execution_engine.join_all_waiting_work + if !@interface_thread.alive? + # Join the thread to have it to raise an exception that + # would have terminated it + @interface_thread.join + # If no exception was risen, fail with a less helpful + # message + flunck("interface thread quit unexpectedly") + else + @interface_thread_sync.wait + @interface_thread.join + end + ensure + @interface_thread = nil + end + end + end +end diff --git a/test/test_shell_interface.rb b/test/test_shell_interface.rb deleted file mode 100644 index 716a48769..000000000 --- a/test/test_shell_interface.rb +++ /dev/null @@ -1,249 +0,0 @@ -# frozen_string_literal: true - -require "syskit/test/self" -require "syskit/shell_interface" - -module Syskit - describe ShellInterface do - attr_reader :subject - - before do - @subject = ShellInterface.new(flexmock(plan: plan)) - subject.execution_engine.thread = Thread.current - @interface_thread = nil - end - - after do - @ee_thread&.join - end - - describe "#redeploy" do - it "triggers a full deployment" do - flexmock(Runtime).should_receive(:apply_requirement_modifications) - .with(subject.plan, force: true).once.pass_thru - subject.redeploy - end - end - - describe "#restart_deployments" do - attr_reader :task_m, :task - before do - @task_m = TaskContext.new_submodel - @task = syskit_stub_deploy_configure_and_start( - syskit_stub_requirements(task_m).with_conf("default") - ) - plan.add_mission_task(task) - end - - it "stops the matching deployments and redeploys" do - plug_apply_requirement_modifications - expect_execution do - subject.restart_deployments - end.to do - emit find_tasks(ShellInterface::ShellDeploymentRestart).stop_event - end - assert_equal 1, plan.find_tasks(task_m).pending.to_a.size - end - - it "restricts the deployments to the given models" do - other = syskit_stub_deploy_configure_and_start( - syskit_stub_requirements(TaskContext.new_submodel) - ) - plug_apply_requirement_modifications - expect_execution do - subject.restart_deployments(task.execution_agent.model) - end.to do - emit find_tasks(ShellInterface::ShellDeploymentRestart).stop_event - emit task.stop_event - not_emit other.stop_event - end - assert_equal 1, plan.find_tasks(task_m).pending.to_a.size - end - - it "accepts task models as argument" do - other = syskit_stub_deploy_configure_and_start( - syskit_stub_requirements(TaskContext.new_submodel) - ) - plug_apply_requirement_modifications - expect_execution do - subject.restart_deployments(task.model) - end.to do - emit find_tasks(ShellInterface::ShellDeploymentRestart).stop_event - emit task.stop_event - not_emit other.stop_event - end - assert_equal 1, plan.find_tasks(task_m).pending.to_a.size - end - end - - describe "#stop_deployments" do - attr_reader :task_m, :task - before do - @task_m = TaskContext.new_submodel - @task = syskit_stub_deploy_configure_and_start(task_m.with_conf("default")) - plan.add_mission_task(task) - end - - it "stops the matching deployments" do - expect_execution { subject.stop_deployments } - .to do - emit task.aborted_event - emit task.execution_agent.stop_event - end - assert task.finished? - end - - it "restricts the deployments to the given models" do - other = syskit_stub_deploy_configure_and_start(task_m.with_conf("other")) - subject.plan.add_mission_task(other) - expect_execution { subject.stop_deployments(task.execution_agent.model) } - .to do - emit task.aborted_event - emit task.execution_agent.stop_event - end - assert task.finished? - assert !other.finished? - end - - it "accepts task models as argument" do - other_m = TaskContext.new_submodel - other = syskit_stub_deploy_configure_and_start(other_m) - subject.plan.add_mission_task(other) - expect_execution { subject.stop_deployments(task.execution_agent.model) } - .to do - emit task.aborted_event - emit task.execution_agent.stop_event - end - assert task.finished? - assert !other.finished? - end - end - - describe "the log configuration management" do - before { Syskit.conf.logs.create_group "test" } - after { Syskit.conf.logs.remove_group("test") } - - it "creates a marshallable instance of the configuration" do - conf = subject.logging_conf - assert_equal conf.port_logs_enabled, Syskit.conf.logs.port_logs_enabled? - assert_equal conf.conf_logs_enabled, Syskit.conf.logs.conf_logs_enabled? - Syskit.conf.logs.groups.each_pair do |key, group| - assert_equal group.enabled?, conf.groups[key].enabled - end - Marshal.dump(conf) - end - - it "changes status of conf and port logging and redeploys" do - conf = subject.logging_conf - previous_port_status = Syskit.conf.logs.port_logs_enabled? - previous_conf_status = Syskit.conf.logs.conf_logs_enabled? - - conf.port_logs_enabled = !previous_port_status - conf.conf_logs_enabled = !previous_conf_status - - flexmock(subject).should_receive(:redeploy).once.pass_thru do - assert_equal Syskit.conf.logs.port_logs_enabled?, !previous_port_status - assert_equal Syskit.conf.logs.conf_logs_enabled?, !previous_conf_status - end - subject.update_logging_conf(conf) - end - - it "changes status of an existing log group and redeploys" do - conf = subject.logging_conf - previous_status = Syskit.conf.logs.group_by_name("test").enabled? - conf.groups["test"].enabled = !previous_status - - flexmock(subject).should_receive(:redeploy).once.pass_thru do - assert_equal Syskit.conf.logs.group_by_name("test").enabled?, !previous_status - end - subject.update_logging_conf(conf) - end - end - - describe "the log group management" do - attr_reader :group - before do - @group = Syskit.conf.logs.create_group "test" do |g| - g.add /base.samples.frame.Frame/ - end - end - - after do - Syskit.conf.logs.remove_group("test") - end - - it "enable_log_group enables the log group and redeploys" do - group.enabled = false - flexmock(subject).should_receive(:redeploy).once.ordered - subject.enable_log_group "test" - assert group.enabled? - end - - it "disable_log_group enables the log group and redeploys" do - group.enabled = true - flexmock(subject).should_receive(:redeploy).once.ordered - subject.disable_log_group "test" - assert !group.enabled? - end - - it "enable_log_group raises ArgumentError if the log group does not exist" do - assert_raises(ArgumentError) do - subject.enable_log_group "does_not_exist" - end - end - - it "disable_log_group raises ArgumentError if the log group does not exist" do - assert_raises(ArgumentError) do - subject.disable_log_group "does_not_exist" - end - end - end - - # Start a thread to do a call on the interface that is synchronized - # with {ExecutionEngine#execute} - # - # @example call the 'redeploy' interface command, and wait for its - # result - # - # # Call 'redeploy'. The queue method waits 50ms to give time to - # # the thread to actually start the call. There's no way to be - # # sure, so that might lead to random test failures - # queue_execute_call { subject.redeploy } - # # Force processing of the call, and wait for the thread to - # # finish - # process_execute_call - def queue_execute_call(&block) - if @interface_thread - raise "you must call #process_execute_call after a call "\ - "to #queue_execute_call" - end - - @interface_thread_sync = sync = Concurrent::CyclicBarrier.new(2) - @interface_thread = Thread.new do - sync.wait - block.call - sync.wait - end - sync.wait - sleep 0.05 - end - - # Process the work queued with {#queue_execute_call} - def process_execute_call - subject.execution_engine.join_all_waiting_work - if !@interface_thread.alive? - # Join the thread to have it to raise an exception that - # would have terminated it - @interface_thread.join - # If no exception was risen, fail with a less helpful - # message - flunck("interface thread quit unexpectedly") - else - @interface_thread_sync.wait - @interface_thread.join - end - ensure - @interface_thread = nil - end - end -end From acb2b473d9b7c2fbf9ef66c2adc708a5f2468b31 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 4 Jul 2023 11:06:07 -0300 Subject: [PATCH 153/260] feat: implement Interface::Commands#deployments --- lib/syskit/interface.rb | 1 + lib/syskit/interface/commands.rb | 11 +++++++++++ lib/syskit/interface/protocol.rb | 29 +++++++++++++++++++++++++++++ test/interface/test_commands.rb | 20 ++++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 lib/syskit/interface/protocol.rb diff --git a/lib/syskit/interface.rb b/lib/syskit/interface.rb index ef6098227..43e879917 100644 --- a/lib/syskit/interface.rb +++ b/lib/syskit/interface.rb @@ -1,3 +1,4 @@ # frozen_string_literal: true +require "syskit/interface/protocol" require "syskit/interface/commands" \ No newline at end of file diff --git a/lib/syskit/interface/commands.rb b/lib/syskit/interface/commands.rb index 547f6a3e2..8df925694 100644 --- a/lib/syskit/interface/commands.rb +++ b/lib/syskit/interface/commands.rb @@ -7,6 +7,17 @@ module Syskit module Interface # Definition of the syskit-specific interface commands class Commands < Roby::Interface::CommandLibrary + # Return information about deployments + # + # @return [Protocol::Deployment] + def deployments + plan.find_tasks(Syskit::Deployment).map do |task| + Protocol.marshal_deployment_task(task) + end + end + command :deployments, + "returns information about running deployments" + # Save the configuration of all running tasks of the given model to disk # # @param [String,nil] name the section name for the new configuration. diff --git a/lib/syskit/interface/protocol.rb b/lib/syskit/interface/protocol.rb new file mode 100644 index 000000000..8e0d62f7d --- /dev/null +++ b/lib/syskit/interface/protocol.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require "roby/interface/protocol" + +module Syskit + module Interface + # Syskit extensions to Roby's interface wire protocol + module Protocol + Deployment = Struct.new :name, :state, :on, :pid, :ready_since, + :task_id, :iors, keyword_init: true + + def self.marshal_deployment_task(task) + if (ready_since = task.ready_event.last&.time) + ready_since = ready_since.tv_sec + end + + Protocol::Deployment.new( + name: task.process_name, + state: task.current_state, + on: task.arguments[:on], + pid: task.pid, + ready_since: ready_since, + task_id: task.droby_id.id, + iors: task.remote_task_handles.transform_values { _1.handle.ior } + ) + end + end + end +end diff --git a/test/interface/test_commands.rb b/test/interface/test_commands.rb index df5aa119a..9881a4fc7 100644 --- a/test/interface/test_commands.rb +++ b/test/interface/test_commands.rb @@ -26,6 +26,26 @@ module Interface end end + describe "#deployments" do + attr_reader :task_m, :task + before do + @task_m = TaskContext.new_submodel + @task = syskit_stub_deploy_configure_and_start( + syskit_stub_requirements(task_m).with_conf("default") + ) + plan.add_mission_task(task) + end + + it "returns the list of deployments" do + deployments = subject.deployments + assert_equal 1, deployments.size + deployment = deployments.first + assert_kind_of Deployment, deployment + assert_equal ::Process.pid, deployment.pid + assert_equal "stubs", deployment.arguments[:on] + end + end + describe "#restart_deployments" do attr_reader :task_m, :task before do From d9a6374762f6be049dfc8f025dcf39766dc55de6 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 28 Mar 2024 17:41:25 -0300 Subject: [PATCH 154/260] fix: do the proper integration of v2 protocol --- lib/syskit/interface.rb | 3 +-- lib/syskit/interface/commands.rb | 4 +-- lib/syskit/interface/protocol.rb | 29 --------------------- lib/syskit/interface/v2/protocol.rb | 39 +++++++++++++++++++++++++++++ lib/syskit/roby_app/plugin.rb | 8 ++++++ 5 files changed, 49 insertions(+), 34 deletions(-) delete mode 100644 lib/syskit/interface/protocol.rb create mode 100644 lib/syskit/interface/v2/protocol.rb diff --git a/lib/syskit/interface.rb b/lib/syskit/interface.rb index 43e879917..3135af052 100644 --- a/lib/syskit/interface.rb +++ b/lib/syskit/interface.rb @@ -1,4 +1,3 @@ # frozen_string_literal: true -require "syskit/interface/protocol" -require "syskit/interface/commands" \ No newline at end of file +require "syskit/interface/commands" diff --git a/lib/syskit/interface/commands.rb b/lib/syskit/interface/commands.rb index 8df925694..dc0d8eeca 100644 --- a/lib/syskit/interface/commands.rb +++ b/lib/syskit/interface/commands.rb @@ -11,9 +11,7 @@ class Commands < Roby::Interface::CommandLibrary # # @return [Protocol::Deployment] def deployments - plan.find_tasks(Syskit::Deployment).map do |task| - Protocol.marshal_deployment_task(task) - end + plan.find_tasks(Syskit::Deployment).to_a end command :deployments, "returns information about running deployments" diff --git a/lib/syskit/interface/protocol.rb b/lib/syskit/interface/protocol.rb deleted file mode 100644 index 8e0d62f7d..000000000 --- a/lib/syskit/interface/protocol.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -require "roby/interface/protocol" - -module Syskit - module Interface - # Syskit extensions to Roby's interface wire protocol - module Protocol - Deployment = Struct.new :name, :state, :on, :pid, :ready_since, - :task_id, :iors, keyword_init: true - - def self.marshal_deployment_task(task) - if (ready_since = task.ready_event.last&.time) - ready_since = ready_since.tv_sec - end - - Protocol::Deployment.new( - name: task.process_name, - state: task.current_state, - on: task.arguments[:on], - pid: task.pid, - ready_since: ready_since, - task_id: task.droby_id.id, - iors: task.remote_task_handles.transform_values { _1.handle.ior } - ) - end - end - end -end diff --git a/lib/syskit/interface/v2/protocol.rb b/lib/syskit/interface/v2/protocol.rb new file mode 100644 index 000000000..6da2e81be --- /dev/null +++ b/lib/syskit/interface/v2/protocol.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Syskit + module Interface + module V2 + # Syskit extensions to Roby's v2 interface wire protocol + module Protocol + Deployment = Struct.new( + :roby_task, :pid, :ready_since, :iors, keyword_init: true + ) do + def pretty_print(pp) + roby_task.pretty_print(pp) + pp.breakable + pp.text "PID: #{pid}" + pp.breakable + pp.text "Deployed tasks: #{iors.keys.join(', ')}" + end + end + + def self.register_marshallers(protocol) + protocol.add_marshaller( + Syskit::Deployment, &method(:marshal_deployment_task) + ) + end + + def self.marshal_deployment_task(channel, task) + Deployment.new( + roby_task: Roby::Interface::V2::Protocol.marshal_task( + channel, task + ), + pid: task.pid, + ready_since: task.ready_event.last&.time, + iors: task.remote_task_handles.transform_values { _1.handle.ior } + ) + end + end + end + end +end diff --git a/lib/syskit/roby_app/plugin.rb b/lib/syskit/roby_app/plugin.rb index 50cb6f0f9..4ace9c942 100644 --- a/lib/syskit/roby_app/plugin.rb +++ b/lib/syskit/roby_app/plugin.rb @@ -958,6 +958,14 @@ def self.enable toplevel_object.extend LoadToplevelMethods end + # Plugin hook called by Roby to setup the v2 interface protocol + def self.setup_interface_v2_protocol + require "syskit/interface/v2/protocol" + Syskit::Interface::V2::Protocol.register_marshallers( + Roby::Interface::V2::Protocol + ) + end + class VariableSizedType < RuntimeError; end def self.validate_port_has_fixed_size(port, with_global_size, only_warn: false, ignore: []) From 64d72890e3f779b63b049148d4483a33a28c02d4 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sat, 6 Apr 2024 11:04:45 -0300 Subject: [PATCH 155/260] fix: tests --- test/gui/test_logging_configuration.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/gui/test_logging_configuration.rb b/test/gui/test_logging_configuration.rb index 45d6bbfbb..b30c2ce4a 100644 --- a/test/gui/test_logging_configuration.rb +++ b/test/gui/test_logging_configuration.rb @@ -11,9 +11,9 @@ module GUI attr_reader :client attr_reader :conf before do - @conf = Syskit::ShellInterface::LoggingConfiguration.new true, true, {} - @conf.groups["images"] = Syskit::ShellInterface::LoggingGroup.new "images", true - @conf.groups["messages"] = Syskit::ShellInterface::LoggingGroup.new "messages", true + @conf = Interface::Commands::LoggingConfiguration.new true, true, {} + @conf.groups["images"] = Interface::Commands::LoggingGroup.new "images", true + @conf.groups["messages"] = Interface::Commands::LoggingGroup.new "messages", true @client = flexmock("client") @syskit = flexmock("syskit") @@ -105,7 +105,7 @@ def assert_items_modified(items, modified = true) end it "adds group to view" do - conf.groups["events"] = Syskit::ShellInterface::LoggingGroup.new "events", true + conf.groups["events"] = Interface::Commands::LoggingGroup.new "events", true subject.refresh assert_view_matches_conf From 9382101b72e7d7f2af125226338991bcb9c088c7 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 2 Apr 2024 13:39:15 -0300 Subject: [PATCH 156/260] feat: implement `syskit telemetry ui` based on the copy of the current runtime view --- lib/syskit/cli/main.rb | 5 +++ lib/syskit/telemetry/cli.rb | 49 ++++++++++++++++++++++++ lib/syskit/telemetry/ui/runtime_state.rb | 42 +++++++++++++++----- 3 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 lib/syskit/telemetry/cli.rb diff --git a/lib/syskit/cli/main.rb b/lib/syskit/cli/main.rb index aadb4213d..9bf6988ea 100644 --- a/lib/syskit/cli/main.rb +++ b/lib/syskit/cli/main.rb @@ -3,6 +3,7 @@ require "roby/cli/main" require "syskit/cli/gen_main" require "syskit/cli/doc_main" +require "syskit/telemetry/cli" module Syskit module CLI @@ -33,6 +34,10 @@ def orogen_test(*args) Process.exec(syskit_path, "test", "--live", *extra_args, *files, "--", *minitest_args, chdir: workdir) end + + desc "telemetry", + "commands related to monitoring and commanding a running Syskit system" + subcommand "telemetry", Telemetry::CLI end end end diff --git a/lib/syskit/telemetry/cli.rb b/lib/syskit/telemetry/cli.rb new file mode 100644 index 000000000..e014a7a0a --- /dev/null +++ b/lib/syskit/telemetry/cli.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require "roby/interface/base" + +module Syskit + module Telemetry + # Implementation of `syskit telemetry` + class CLI < Thor + desc "ui", + "open a UI to interface with a running Syskit system" + option :host, + type: :string, doc: "host[:port] to connect to", + default: "localhost:#{Roby::Interface::DEFAULT_PORT}" + def ui + roby_setup + host, port = parse_host_port( + options[:host], default_port: Roby::Interface::DEFAULT_PORT + ) + + require "syskit/telemetry/ui/runtime_state" + $qApp.disable_threading # rubocop:disable Style/GlobalVars + + require "syskit/scripts/common" + Syskit::Scripts.run do + end + end + + no_commands do + def roby_setup + Roby.app.using "syskit" + Syskit.conf.only_load_models = true + # We don't need the process server, win some startup time + Syskit.conf.disables_local_process_server = true + Roby.app.ignore_all_load_errors = true + Roby.app.development_mode = false + + Roby.app.auto_load_all = false + Roby.app.auto_load_models = false + end + + def parse_host_port(host_port, default_port:) + host_port += ":#{default_port}" unless /:\d+$/.match?(host_port) + match = /(.*):(\d+)$/.match(host_port) + [match[1], Integer(match[2])] + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/runtime_state.rb b/lib/syskit/telemetry/ui/runtime_state.rb index c5d26ebc5..2c165294b 100644 --- a/lib/syskit/telemetry/ui/runtime_state.rb +++ b/lib/syskit/telemetry/ui/runtime_state.rb @@ -1,8 +1,13 @@ # frozen_string_literal: true +require "Qt" +require "qtwebkit" +require "vizkit" require "syskit" -require "roby/interface/async" -require "roby/interface/async/log" +require "metaruby/gui/exception_view" +require "roby/interface/v2/async" +require "roby/interface/v2/async/log" +require "roby/gui/exception_view" require "syskit/telemetry/ui/logging_configuration" require "syskit/telemetry/ui/job_status_display" require "syskit/telemetry/ui/widget_list" @@ -19,7 +24,7 @@ class RuntimeState < Qt::Widget include Roby::Hooks include Roby::Hooks::InstanceHooks - # @return [Roby::Interface::Async::Interface] the underlying syskit + # @return [Roby::Interface::V2::Async::Interface] the underlying syskit # interface attr_reader :syskit # An async object to access the log stream @@ -117,13 +122,13 @@ def paint(painter, option, index) end end - # @param [Roby::Interface::Async::Interface] syskit the underlying + # @param [Roby::Interface::V2::Async::Interface] syskit the underlying # syskit interface # @param [Integer] poll_period how often should the syskit interface # be polled (milliseconds). Set to nil if the polling is already # done externally def initialize(parent: nil, robot_name: "default", - syskit: Roby::Interface::Async::Interface.new, poll_period: 50) + syskit: Roby::Interface::V2::Async::Interface.new, poll_period: 50) super(parent) @@ -253,7 +258,7 @@ def update_log_server_connection(port) syskit_log_stream.close end - @syskit_log_stream = Roby::Interface::Async::Log.new(syskit.remote_name, port: port) + @syskit_log_stream = Roby::Interface::V2::Async::Log.new(syskit.remote_name, port: port) syskit_log_stream.on_reachable do deselect_job end @@ -596,7 +601,7 @@ def create_ui_new_job def poll_syskit_interface syskit.poll if syskit_log_stream - if syskit_log_stream.poll(max: 0.05) == Roby::Interface::Async::Log::STATE_PENDING_DATA + if syskit_log_stream.poll(max: 0.05) == Roby::Interface::V2::Async::Log::STATE_PENDING_DATA syskit_poll.interval = 0 else syskit_poll.interval = @syskit_poll_period @@ -609,7 +614,7 @@ def poll_syskit_interface # # Create the UI elements for the given job # - # @param [Roby::Interface::Async::JobMonitor] job + # @param [Roby::Interface::V2::Async::JobMonitor] job def monitor_job(job) job_status = JobStatusDisplay.new(job, @batch_manager) job_status_list.add_widget job_status @@ -645,20 +650,37 @@ def select_job(job_status) job_expanded_status.add_tasks_info(all_tasks, all_job_info) end - def restore_from_settings(settings) + def settings + @settings ||= Qt::Settings.new("syskit", "telemetry-ui") + end + + def restore_from_settings(settings = self.settings) %w{ui_hide_loggers ui_show_expanded_job}.each do |checkbox_name| default = Qt::Variant.new(send(checkbox_name).checked) send(checkbox_name).checked = settings.value(checkbox_name, default).to_bool end end - def save_to_settings(settings) + def save_to_settings(settings = self.settings) %w(ui_hide_loggers ui_show_expanded_job).each do |checkbox_name| settings.set_value checkbox_name, Qt::Variant.new(send(checkbox_name).checked) end end signals "fileOpenClicked(const QUrl&)" + + def self.exec(host, port: Roby::Interface::DEFAULT_PORT) + Orocos.initialize + interface = Roby::Interface::V1::Async::Interface.new(host, port: port) + main = UI::RuntimeState.new(syskit: interface) + main.window_title = "Syskit @#{options[:host]}" + + main.restore_from_settings + main.show + Vizkit.exec + main.save_to_settings + main.settings.sync + end end end end From fcd939fecf6078b9e21fe114a93886c15a471e42 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sat, 6 Apr 2024 11:49:14 -0300 Subject: [PATCH 157/260] fix: rubocop --- lib/syskit/telemetry/cli.rb | 19 ++++++++++++++++++- .../telemetry/ui/logging_configuration.rb | 2 +- lib/syskit/telemetry/ui/runtime_state.rb | 13 ------------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/lib/syskit/telemetry/cli.rb b/lib/syskit/telemetry/cli.rb index e014a7a0a..253e86798 100644 --- a/lib/syskit/telemetry/cli.rb +++ b/lib/syskit/telemetry/cli.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true +require "roby" require "roby/interface/base" +require "roby/interface/v1/async" module Syskit module Telemetry @@ -22,10 +24,11 @@ def ui require "syskit/scripts/common" Syskit::Scripts.run do + runtime_state(host, port) end end - no_commands do + no_commands do # rubocop:disable Metrics/BlockLength def roby_setup Roby.app.using "syskit" Syskit.conf.only_load_models = true @@ -43,6 +46,20 @@ def parse_host_port(host_port, default_port:) match = /(.*):(\d+)$/.match(host_port) [match[1], Integer(match[2])] end + + def runtime_state(host, port) + Orocos.initialize + interface = + Roby::Interface::V1::Async::Interface.new(host, port: port) + main = UI::RuntimeState.new(syskit: interface) + main.window_title = "Syskit @#{options[:host]}" + + main.restore_from_settings + main.show + Vizkit.exec + main.save_to_settings + main.settings.sync + end end end end diff --git a/lib/syskit/telemetry/ui/logging_configuration.rb b/lib/syskit/telemetry/ui/logging_configuration.rb index fa232cd7d..90f080642 100644 --- a/lib/syskit/telemetry/ui/logging_configuration.rb +++ b/lib/syskit/telemetry/ui/logging_configuration.rb @@ -4,8 +4,8 @@ require "vizkit/vizkit_items" require "vizkit/tree_view" require "Qt4" -require "syskit/interface" require "syskit/telemetry/ui/logging_configuration_item" +require "roby/interface" require "roby/interface/exceptions" module Syskit diff --git a/lib/syskit/telemetry/ui/runtime_state.rb b/lib/syskit/telemetry/ui/runtime_state.rb index 2c165294b..bbbd2769a 100644 --- a/lib/syskit/telemetry/ui/runtime_state.rb +++ b/lib/syskit/telemetry/ui/runtime_state.rb @@ -668,19 +668,6 @@ def save_to_settings(settings = self.settings) end signals "fileOpenClicked(const QUrl&)" - - def self.exec(host, port: Roby::Interface::DEFAULT_PORT) - Orocos.initialize - interface = Roby::Interface::V1::Async::Interface.new(host, port: port) - main = UI::RuntimeState.new(syskit: interface) - main.window_title = "Syskit @#{options[:host]}" - - main.restore_from_settings - main.show - Vizkit.exec - main.save_to_settings - main.settings.sync - end end end end From be991e3b64d93936a3560dba49d0e9660d7d195f Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 7 Apr 2024 07:29:48 -0300 Subject: [PATCH 158/260] feat: add GPRC scaffold --- .gitignore | 2 + .rubocop.yml | 2 + Rakefile | 20 ++++-- lib/syskit/telemetry/agent/agent.proto | 97 ++++++++++++++++++++++++++ lib/syskit/telemetry/agent/client.rb | 12 ++++ lib/syskit/telemetry/agent/server.rb | 12 ++++ manifest.xml | 4 ++ test/telemetry/agent/test_client.rb | 13 ++++ test/telemetry/agent/test_server.rb | 13 ++++ 9 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 lib/syskit/telemetry/agent/agent.proto create mode 100644 lib/syskit/telemetry/agent/client.rb create mode 100644 lib/syskit/telemetry/agent/server.rb create mode 100644 test/telemetry/agent/test_client.rb create mode 100644 test/telemetry/agent/test_server.rb diff --git a/.gitignore b/.gitignore index 058cd6d3b..39a4d7081 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ coverage/ tmp/ syskit-0.1.gem + +lib/syskit/telemetry/agent/*_pb.rb diff --git a/.rubocop.yml b/.rubocop.yml index 20adaae4d..592427a69 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -10,6 +10,8 @@ inherit_mode: AllCops: TargetRubyVersion: "2.5" + Exclude: + - lib/syskit/telemetry/agent/*_pb.rb Style/MultilineMemoization: EnforcedStyle: braces diff --git a/Rakefile b/Rakefile index 4b929efdd..9aa4339c2 100644 --- a/Rakefile +++ b/Rakefile @@ -10,6 +10,7 @@ TESTOPTS = ENV.delete("TESTOPTS") || "" RUBOCOP_REQUIRED = (ENV["RUBOCOP"] == "1") USE_RUBOCOP = (ENV["RUBOCOP"] != "0") USE_JUNIT = (ENV["JUNIT"] == "1") +USE_GRPC = (ENV["SYSKIT_HAS_GRPC"] != "0") REPORT_DIR = ENV["REPORT_DIR"] || File.expand_path("test_reports", __dir__) def minitest_set_options(test_task, name) @@ -76,12 +77,19 @@ if USE_RUBOCOP end end -begin - require "coveralls/rake/task" - Coveralls::RakeTask.new - task "test:coveralls" => ["test", "coveralls:push"] -rescue LoadError # rubocop:disable Lint/SuppressedException -end +protogen = + file "lib/syskit/telemetry/agent/agent_pb.rb" => + ["lib/syskit/telemetry/agent/agent.proto"] do + system( + "grpc_tools_ruby_protoc", + "syskit/telemetry/agent/agent.proto", + "--ruby_out=.", + "--grpc_out=.", + chdir: "lib", + exception: true + ) + end +task "default" => protogen if USE_GRPC # For backward compatibility with some scripts that expected hoe task "gem" => "build" diff --git a/lib/syskit/telemetry/agent/agent.proto b/lib/syskit/telemetry/agent/agent.proto new file mode 100644 index 000000000..11f8ae56b --- /dev/null +++ b/lib/syskit/telemetry/agent/agent.proto @@ -0,0 +1,97 @@ +package syskit.telemetry.agent.grpc; + +/** Protocol definition for Syskit's telemetry agent + * + * The objective of the agent is to make it viable to monitor a complete syskit system + * over a bandwidth-limited, high latency, communication channel. The purpose of this + * interface is to reduce the amount of request/replies needed to get an interface + * looking like the syskit IDE runtime view. + * + * Data monitoring is done a subscription basis. The client must first create a permanent + * streaming connection using data() and then subscribes or removes subscriptions + * using the *MonitoringStart and *MonitoringStop calls. + * + * The data call is what maintains the subscriptions. If the call is interrupted or + * cancelled, the whole subscription process must be re-done. + */ +service Server { + /** Establish a connection + * + * This creates a data connection, whose ID will be used in other calls to declare + * what will be written. This must be done before calling the MonitoringStart + * and MonitoringStop functions + * + * Cancel the call to kill the connection + */ + rpc data(Void) returns (stream DataStreamValue) {} + + /** Add ports to a connection data stream + * + * The connection must be alive. The call returns the type definitions + * (as typelib XML) and unique numerical IDs that allow to recognize the ports + * on the data connection. The stream definitions are returned in the same order + * they were given + */ + rpc portMonitoringStart(PortMonitors) returns (DataStreams) {} + + /** Add ports to a connection data stream + * + * The connection must be alive. The call returns the type definitions + * (as typelib XML) and unique numerical IDs that allow to recognize the ports + * on the data connection. The stream definitions are returned in the same order + * they were given + */ + rpc portMonitoringStop(PortMonitorIDs) returns (Void) {} +} + +message Void { +} + +message Error { + required int32 code = 1; + required string message = 2; +} + +enum BufferType { + DATA = 0; + BUFFER_DROP_OLD = 1; + BUFFER_DROP_NEW = 2; +} + +message BufferPolicy { + required BufferType type = 1; + optional uint32 size = 2; +} + +message PortMonitor { + required string task_name = 1; + required string port_name = 2; + required float period = 3; + required BufferPolicy policy = 4; +} + +message PortMonitors { + repeated PortMonitor monitors = 1; +} + +message PortMonitorIDs { + repeated uint32 ids = 1; +} + +message DataStreamValue { + required int32 id = 1; + required bytes data = 2; +} + +message DataStreamValues { + repeated DataStreamValue values = 2; +} + +message DataStream { + required int32 id = 1; + required string typelib_xml = 2; +} + +message DataStreams { + repeated DataStream streams = 1; +} \ No newline at end of file diff --git a/lib/syskit/telemetry/agent/client.rb b/lib/syskit/telemetry/agent/client.rb new file mode 100644 index 000000000..224202778 --- /dev/null +++ b/lib/syskit/telemetry/agent/client.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require "syskit/telemetry/agent/agent_services_pb" + +module Syskit + module Telemetry + module Agent + class Client < Grpc::Server::Stub + end + end + end +end diff --git a/lib/syskit/telemetry/agent/server.rb b/lib/syskit/telemetry/agent/server.rb new file mode 100644 index 000000000..94befef58 --- /dev/null +++ b/lib/syskit/telemetry/agent/server.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require "syskit/telemetry/agent/agent_services_pb" + +module Syskit + module Telemetry + module Agent + class Server < Grpc::Server::Service + end + end + end +end diff --git a/manifest.xml b/manifest.xml index fb3f99957..0cee9becb 100644 --- a/manifest.xml +++ b/manifest.xml @@ -28,6 +28,10 @@ + + + + diff --git a/test/telemetry/agent/test_client.rb b/test/telemetry/agent/test_client.rb new file mode 100644 index 000000000..5b5bb0c43 --- /dev/null +++ b/test/telemetry/agent/test_client.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require "syskit/test/self" +require "syskit/telemetry/agent/client" + +module Syskit + module Telemetry + module Agent + describe Client do + end + end + end +end diff --git a/test/telemetry/agent/test_server.rb b/test/telemetry/agent/test_server.rb new file mode 100644 index 000000000..66bfc2e56 --- /dev/null +++ b/test/telemetry/agent/test_server.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require "syskit/test/self" +require "syskit/telemetry/agent/server" + +module Syskit + module Telemetry + module Agent + describe Server do + end + end + end +end From 54baff652a81245727e0f78d326f514a16cb1229 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 15 Apr 2024 17:26:46 -0300 Subject: [PATCH 159/260] fix: correct documentation for portMonitoringStop in the telemetry proto --- lib/syskit/telemetry/agent/agent.proto | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/syskit/telemetry/agent/agent.proto b/lib/syskit/telemetry/agent/agent.proto index 11f8ae56b..5e38fa7fc 100644 --- a/lib/syskit/telemetry/agent/agent.proto +++ b/lib/syskit/telemetry/agent/agent.proto @@ -34,12 +34,11 @@ service Server { */ rpc portMonitoringStart(PortMonitors) returns (DataStreams) {} - /** Add ports to a connection data stream + /** Remove ports from a connection data stream * - * The connection must be alive. The call returns the type definitions - * (as typelib XML) and unique numerical IDs that allow to recognize the ports - * on the data connection. The stream definitions are returned in the same order - * they were given + * The ID list given as argument is made of the IDs returned by + * portMonitoringStart. Passing IDs that do not exist or were + * already removed does nothing */ rpc portMonitoringStop(PortMonitorIDs) returns (Void) {} } @@ -94,4 +93,4 @@ message DataStream { message DataStreams { repeated DataStream streams = 1; -} \ No newline at end of file +} From dd78ec91552e3f41a5309922d212c2a753b85eb4 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sat, 6 Apr 2024 14:58:39 -0300 Subject: [PATCH 160/260] feat: use the v2 interface in `telemetry ui` This uses Syskit to resolve orocos tasks, and uses the new protocol to show data. At least, it tremendously speeds up task resolution, and ensures that we see all the tasks Syskit sees (instead of only the tasks that are present on the main computer) --- .rubocop_todo.yml | 1 + lib/syskit/telemetry/cli.rb | 9 +- lib/syskit/telemetry/ui/name_service.rb | 68 +++++ lib/syskit/telemetry/ui/runtime_state.rb | 303 +++++++++++++---------- 4 files changed, 246 insertions(+), 135 deletions(-) create mode 100644 lib/syskit/telemetry/ui/name_service.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 063c97088..c179e2b7b 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -573,6 +573,7 @@ Metrics/ClassLength: - 'lib/syskit/network_generation/engine.rb' - 'lib/syskit/runtime/connection_management.rb' - 'lib/syskit/task_context.rb' + - 'lib/syskit/telemetry/ui/runtime_state.rb' # Offense count: 105 # Configuration parameters: Max. diff --git a/lib/syskit/telemetry/cli.rb b/lib/syskit/telemetry/cli.rb index 253e86798..42c34fcac 100644 --- a/lib/syskit/telemetry/cli.rb +++ b/lib/syskit/telemetry/cli.rb @@ -2,7 +2,6 @@ require "roby" require "roby/interface/base" -require "roby/interface/v1/async" module Syskit module Telemetry @@ -12,11 +11,11 @@ class CLI < Thor "open a UI to interface with a running Syskit system" option :host, type: :string, doc: "host[:port] to connect to", - default: "localhost:#{Roby::Interface::DEFAULT_PORT}" + default: "localhost:#{Roby::Interface::DEFAULT_PORT_V2}" def ui roby_setup host, port = parse_host_port( - options[:host], default_port: Roby::Interface::DEFAULT_PORT + options[:host], default_port: Roby::Interface::DEFAULT_PORT_V2 ) require "syskit/telemetry/ui/runtime_state" @@ -49,8 +48,8 @@ def parse_host_port(host_port, default_port:) def runtime_state(host, port) Orocos.initialize - interface = - Roby::Interface::V1::Async::Interface.new(host, port: port) + interface = Roby::Interface::V2::Async::Interface + .new(host, port: port) main = UI::RuntimeState.new(syskit: interface) main.window_title = "Syskit @#{options[:host]}" diff --git a/lib/syskit/telemetry/ui/name_service.rb b/lib/syskit/telemetry/ui/name_service.rb new file mode 100644 index 000000000..813a1e599 --- /dev/null +++ b/lib/syskit/telemetry/ui/name_service.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Syskit + module Telemetry + module UI + # Copy of Runkit's local name service to use with orocos.rb + class NameService < Orocos::NameServiceBase + # A new NameService instance + # + # @param [Hash] tasks The tasks which are + # known by the name service. + # @note The namespace is always "Local" + def initialize(tasks = []) + @registered_tasks = Concurrent::Hash.new + tasks.each { |task| register(task) } + end + + def names + @registered_tasks.keys + end + + def include?(name) + puts @registered_tasks.keys + @registered_tasks.key?(name) + end + + # (see NameServiceBase#get) + def ior(name) + task = @registered_tasks[name] + return task.ior if task.respond_to?(:ior) + + raise Orocos::NotFound, "task context #{name} cannot be found." + end + + # (see NameServiceBase#get) + def get(name, **) + task = @registered_tasks[name] + return task if task + + raise Orocos::NotFound, "task context #{name} cannot be found." + end + + # Registers the given {Orocos::TaskContext} on the name service. + # If a name is provided, it will be used as an alias. If no name is + # provided, the name of the task is used. This is true even if the + # task name is renamed later. + # + # @param [Orocos::TaskContext] task The task. + # @param [String] name Optional name which is used to register the task. + def register(task, name: task.name) + @registered_tasks[name] = task + end + + # Deregisters the given name or task from the name service. + # + # @param [String,TaskContext] name The name or task + def deregister(name) + @registered_tasks.delete(name) + end + + # (see Base#cleanup) + def cleanup + @registered_tasks.clear + end + end + end + end +end diff --git a/lib/syskit/telemetry/ui/runtime_state.rb b/lib/syskit/telemetry/ui/runtime_state.rb index bbbd2769a..1971ac84e 100644 --- a/lib/syskit/telemetry/ui/runtime_state.rb +++ b/lib/syskit/telemetry/ui/runtime_state.rb @@ -6,7 +6,6 @@ require "syskit" require "metaruby/gui/exception_view" require "roby/interface/v2/async" -require "roby/interface/v2/async/log" require "roby/gui/exception_view" require "syskit/telemetry/ui/logging_configuration" require "syskit/telemetry/ui/job_status_display" @@ -15,6 +14,8 @@ require "syskit/telemetry/ui/global_state_label" require "syskit/telemetry/ui/app_start_dialog" require "syskit/telemetry/ui/batch_manager" +require "syskit/telemetry/ui/name_service" +require "syskit/interface/v2" module Syskit module Telemetry @@ -27,8 +28,6 @@ class RuntimeState < Qt::Widget # @return [Roby::Interface::V2::Async::Interface] the underlying syskit # interface attr_reader :syskit - # An async object to access the log stream - attr_reader :syskit_log_stream # The toplevel layout attr_reader :main_layout @@ -48,11 +47,6 @@ class RuntimeState < Qt::Widget # state attr_reader :connection_state - # All known tasks - attr_reader :all_tasks - # Job information for tasks in the rebuilt plan - attr_reader :all_job_info - # The name service which allows us to resolve Rock task contexts attr_reader :name_service # A task inspector widget we use to display the task states @@ -141,9 +135,7 @@ def initialize(parent: nil, robot_name: "default", connect syskit_poll, SIGNAL("timeout()"), self, SLOT("poll_syskit_interface()") - if poll_period - @syskit_poll.start(poll_period) - end + @syskit_poll.start(poll_period) if poll_period create_ui @@ -166,15 +158,15 @@ def initialize(parent: nil, robot_name: "default", @current_job = nil @current_orocos_tasks = Set.new - @all_tasks = Set.new - @known_loggers = nil - @all_job_info = {} + @proxies = {} + syskit.on_ui_event do |event_name, *args| - if w = @ui_event_widgets[event_name] + if (w = @ui_event_widgets[event_name]) w.show w.update(*args) else - puts "don't know what to do with UI event #{event_name}, known events: #{@ui_event_widgets}" + puts "don't know what to do with UI event #{event_name}, "\ + "known events: #{@ui_event_widgets}" end end on_connection_state_changed do |state| @@ -183,7 +175,6 @@ def initialize(parent: nil, robot_name: "default", end syskit.on_reachable do @syskit_commands = syskit.client.syskit - update_log_server_connection(syskit.log_server_port) @job_status_list.each_widget do |w| w.show_actions = true end @@ -192,7 +183,9 @@ def initialize(parent: nil, robot_name: "default", syskit.actions.sort_by(&:name).each do |action| next if action.advanced? - action_combo.add_item(action.name, Qt::Variant.new(action.doc)) + action_combo.add_item( + action.name, Qt::Variant.new(action.doc) + ) end ui_logging_configuration.refresh global_actions[:start].visible = false @@ -247,37 +240,11 @@ def monitor_syskit_startup def reset Orocos.initialize @logger_m = nil - orocos_corba_nameservice = Orocos::CORBA::NameService.new(syskit.remote_name) - @name_service = Orocos::Async::NameService.new(orocos_corba_nameservice) - end - - def update_log_server_connection(port) - if syskit_log_stream && (syskit_log_stream.port == port) - return - elsif syskit_log_stream - syskit_log_stream.close - end + @call_guards = {} + @orogen_models = {} - @syskit_log_stream = Roby::Interface::V2::Async::Log.new(syskit.remote_name, port: port) - syskit_log_stream.on_reachable do - deselect_job - end - syskit_log_stream.on_init_progress do |rx, expected| - run_hook :on_progress, format("loading %02i", Float(rx) / expected * 100) - end - syskit_log_stream.on_update do |cycle_index, cycle_time| - if syskit_log_stream.init_done? - time_s = cycle_time.strftime("%H:%M:%S.%3N").to_s - run_hook :on_progress, format("@%i %s", cycle_index, time_s) - - job_expanded_status.update_time(cycle_index, cycle_time) - update_tasks_info - job_expanded_status.add_tasks_info(all_tasks, all_job_info) - job_expanded_status.scheduler_state = syskit_log_stream.scheduler_state - job_expanded_status.update_chronicle unless hide_expanded_jobs? - end - syskit_log_stream.clear_integrated - end + @name_service = NameService.new + @async_name_service = Orocos::Async::NameService.new(@name_service) end def hide_loggers? @@ -334,68 +301,6 @@ def logger_task?(t) t.kind_of?(@logger_m) if @logger_m end - def update_tasks_info - if current_job - job_task = syskit_log_stream.plan.find_tasks(Roby::Interface::Job) - .with_arguments(job_id: current_job.job_id) - .first - return unless job_task - - placeholder_task = job_task.planned_task || job_task - return unless placeholder_task - - dependency = placeholder_task.relation_graph_for(Roby::TaskStructure::Dependency) - tasks = dependency.enum_for(:depth_first_visit, placeholder_task).to_a - tasks << job_task - else - tasks = syskit_log_stream.plan.tasks - end - - if hide_loggers? - unless @known_loggers - @known_loggers = Set.new - all_tasks.delete_if do |t| - @known_loggers << t if logger_task?(t) - end - end - - tasks = tasks.find_all do |t| - if all_tasks.include?(t) - true - elsif @known_loggers.include?(t) - false - elsif logger_task?(t) - @known_loggers << t - false - else true - end - end - end - all_tasks.merge(tasks) - tasks.each do |job| - if job.kind_of?(Roby::Interface::Job) - placeholder_task = job.planned_task || job - all_job_info[placeholder_task] = job - end - end - update_orocos_tasks - end - - def update_orocos_tasks - candidate_tasks = all_tasks - .find_all { |t| t.kind_of?(Syskit::TaskContext) } - orocos_tasks = candidate_tasks.map { |t| t.arguments[:orocos_name] }.compact.to_set - removed = current_orocos_tasks - orocos_tasks - new = orocos_tasks - current_orocos_tasks - removed.each do |task_name| - ui_task_inspector.remove_task(task_name) - end - new.each do |task_name| - ui_task_inspector.add_task(name_service.proxy(task_name)) - end - @current_orocos_tasks = orocos_tasks - end - EventWidget = Struct.new :name, :widget, :hook do def show widget.show @@ -479,7 +384,6 @@ def create_ui ) @ui_hide_loggers.checked = false @ui_hide_loggers.connect SIGNAL("toggled(bool)") do |checked| - @known_loggers = nil update_tasks_info end @ui_show_expanded_job.checked = true @@ -599,23 +503,173 @@ def create_ui_new_job # # Sets up polling on a given syskit interface def poll_syskit_interface + if syskit.connected? + begin + display_current_cycle_index_and_time + update_current_deployments + update_current_job_task_names if current_job + rescue Roby::Interface::ComError # rubocop:disable Lint/SuppressedException + end + else + reset_current_deployments + reset_current_job + reset_name_service + reset_task_inspector + end + syskit.poll - if syskit_log_stream - if syskit_log_stream.poll(max: 0.05) == Roby::Interface::V2::Async::Log::STATE_PENDING_DATA - syskit_poll.interval = 0 + end + slots "poll_syskit_interface()" + + def display_current_cycle_index_and_time + return unless syskit.cycle_start_time + + time_s = syskit.cycle_start_time.strftime("%H:%M:%S.%3N").to_s + progress_s = format( + "@%i %