Hello, > On 2 Dec 2021, at 16:30, Leo Hofmann wrote: > > Hi, > > Am 02.12.2021 um 16:59 schrieb Michael Tremer: >> Hello, >> >>> On 2 Dec 2021, at 15:39, Leo-Andres Hofmann wrote: >>> >>> Currently the page becomes unresponsive while Pakfire is busy. >>> This patch implements a AJAX/JSON driven log output, to provide >>> continuous information to the user while Pakfire is running. >>> >>> The output is updated 1x per second, if the load should be too high, >>> the interval can be change by writing to "pakfire.refreshInterval". >>> >>> Signed-off-by: Leo-Andres Hofmann >>> --- >>> html/cgi-bin/pakfire.cgi | 153 ++++++++++++++++++---- >>> html/html/include/pakfire.js | 241 +++++++++++++++++++++++++++++++++++ >>> 2 files changed, 368 insertions(+), 26 deletions(-) >>> create mode 100644 html/html/include/pakfire.js >>> >>> diff --git a/html/cgi-bin/pakfire.cgi b/html/cgi-bin/pakfire.cgi >>> index 7957bc154..e5f5f7d6a 100644 >>> --- a/html/cgi-bin/pakfire.cgi >>> +++ b/html/cgi-bin/pakfire.cgi >>> @@ -36,8 +36,11 @@ my %color = (); >>> my %pakfiresettings = (); >>> my %mainsettings = (); >>> >>> -&Header::showhttpheaders(); >>> +# Load general settings >>> +&General::readhash("${General::swroot}/main/settings", \%mainsettings); >>> +&General::readhash("/srv/web/ipfire/html/themes/ipfire/include/colors.txt", \%color); >>> >>> +# Get CGI request data >>> $cgiparams{'ACTION'} = ''; >>> $cgiparams{'VALID'} = ''; >>> >>> @@ -46,12 +49,102 @@ $cgiparams{'DELPAKS'} = ''; >>> >>> &Header::getcgihash(\%cgiparams); >>> >>> -&General::readhash("${General::swroot}/main/settings", \%mainsettings); >>> -&General::readhash("/srv/web/ipfire/html/themes/ipfire/include/colors.txt", \%color); >>> +### Process AJAX/JSON request ### >>> +if($cgiparams{'ACTION'} eq 'json-getstatus') { >>> + # Send HTTP headers >>> + _start_json_output(); >>> + >>> + # Collect Pakfire status and log messages >>> + my %status = ( >>> + 'running' => &_is_pakfire_busy() || "0", >>> + 'running_since' => &General::age("$Pakfire::lockfile") || "0s", >>> + 'reboot' => (-e "/var/run/need_reboot") || "0" >>> + ); >>> + my @messages = `tac /var/log/messages | sed -n '/pakfire:/{p;/Pakfire.*started/q}'`; >>> + >>> + # Start JSON file >>> + print "{\n"; >>> + >>> + foreach my $key (keys %status) { >>> + my $value = $status{$key}; >>> + print qq{\t"$key": "$value",\n}; >>> + } >>> + >>> + # Print sanitized messages in reverse order to undo previous "tac" >>> + print qq{\t"messages": [\n}; >>> + for my $index (reverse (0 .. $#messages)) { >>> + my $line = $messages[$index]; >>> + $line =~ s/[[:cntrl:]<>&\\]+//g; >>> + >>> + print qq{\t\t"$line"}; >>> + print ",\n" unless $index < 1; >>> + } >>> + print "\n\t]\n"; >> What is the reason to “tac” the log file first and then reverse the order again? >> >> Is it just to limit the length of the JSON array? >> >> It might be faster to read the entire file, grep out what we need and then throw away most of the array. Or push a line to the end of the array and remove one from the beginning if it is longer than a certain threshold. > > I wanted to make sure that only the output of the current Pakfire run is shown. Therefore, I use tac and sed to read the logfile backwards until the last "Pakfire ... started!" header is reached. > This works very well, but then of course the messages array is also in reverse order. Okay, that makes sense. Maybe we should start logging things into a separate file to make things easier? /var/log/messages can become really large. A C program is probably the fastest that we could ever have, so might be good enough as a solution for me. -Michael > > All the ideas I had required some form of "reverse", or I had to load the entire file in Perl and check every line. I assumed that tac & sed would be more efficient than any Perl solution I could come up with. I'll try to time this and report back! > > Leo > >> >>> + >>> + # Finalize JSON file & stop >>> + print "}"; >>> + exit; >>> +} >>> + >>> +### Start pakfire page ### >>> +&Header::showhttpheaders(); >>> + >>> +###--- HTML HEAD ---### >>> +my $extraHead = <>> + >>> + >>> + >>> + >>> +END >>> +; >>> +###--- END HTML HEAD ---### >>> + >>> +&Header::openpage($Lang::tr{'pakfire configuration'}, 1, $extraHead); >>> &Header::openbigbox('100%', 'left', '', $errormessage); >>> >>> +# Process Pakfire commands >>> if (($cgiparams{'ACTION'} eq 'install') && (! &_is_pakfire_busy())) { >>> my @pkgs = split(/\|/, $cgiparams{'INSPAKS'}); >>> if ("$cgiparams{'FORCE'}" eq "on") { >>> @@ -170,29 +263,30 @@ if ($errormessage) { >>> &Header::closebox(); >>> } >>> >>> -# Check if pakfire is already running. >>> -if (&_is_pakfire_busy()) { >>> - &Header::openbox( 'Waiting', 1, "" ); >>> - print <>> - >>> -
>>> - $Lang::tr{  >>> - >>> - $Lang::tr{'pakfire working'} >>> -
>>> -
>>> - >>> -
>>> -
>>> -END >>> - my @output = `grep pakfire /var/log/messages | tail -20`; >>> - foreach (@output) { >>> - print "$_
"; >>> - } >>> - print <>> - >>> -
>>> +# Show log output while Pakfire is running >>> +if(&_is_pakfire_busy()) { >>> + &Header::openbox("100%", "center", "Pakfire"); >>> + >>> + print <>> +
>>> +
$Lang::tr{'active'}
>>> +
>>> + $Lang::tr{'pakfire working'}
>>> +
>>> + >>> +
>>> +
$Lang::tr{'refresh'}
>>> +
>>> + >>> + >>> +

>>> +
>>> +
>>> END
>>> +;
>>> +
>>> 	&Header::closebox();
>>> 	&Header::closebigbox();
>>> 	&Header::closepage();
>>> @@ -320,3 +414,10 @@ sub _is_pakfire_busy {
>>> 	# Test presence of PID or lockfile
>>> 	return (($pakfire_pid) || (-e "$Pakfire::lockfile"));
>>> }
>>> +
>>> +# Send HTTP headers
>>> +sub _start_json_output {
>>> +	print "Cache-Control: no-cache, no-store\n";
>>> +	print "Content-Type: application/json\n";
>>> +	print "\n"; # End of HTTP headers
>>> +}
>>> diff --git a/html/html/include/pakfire.js b/html/html/include/pakfire.js
>>> new file mode 100644
>>> index 000000000..0950870e0
>>> --- /dev/null
>>> +++ b/html/html/include/pakfire.js
>>> @@ -0,0 +1,241 @@
>>> +/*#############################################################################
>>> +#                                                                             #
>>> +# IPFire.org - A linux based firewall                                         #
>>> +# Copyright (C) 2007-2021  IPFire Team                       #
>>> +#                                                                             #
>>> +# This program is free software: you can redistribute it and/or modify        #
>>> +# it under the terms of the GNU General Public License as published by        #
>>> +# the Free Software Foundation, either version 3 of the License, or           #
>>> +# (at your option) any later version.                                         #
>>> +#                                                                             #
>>> +# This program is distributed in the hope that it will be useful,             #
>>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
>>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
>>> +# GNU General Public License for more details.                                #
>>> +#                                                                             #
>>> +# You should have received a copy of the GNU General Public License           #
>>> +# along with this program.  If not, see .       #
>>> +#                                                                             #
>>> +#############################################################################*/
>>> +
>>> +"use strict";
>>> +
>>> +// Pakfire Javascript functions (requires jQuery)
>>> +class PakfireJS {
>>> +	constructor() {
>>> +		//--- Public properties ---
>>> +		// Translation strings
>>> +		this.i18n = new PakfireI18N();
>>> +
>>> +		//--- Private properties ---
>>> +		// Status flags (access outside constructor only with setter/getter)
>>> +		this._states = Object.create(null);
>>> +		this._states.running = false;
>>> +		this._states.reboot = false;
>>> +
>>> +		// Status refresh helper
>>> +		this._autoRefresh = {
>>> +			delay: 1000, //Delay between requests (default: 1s)
>>> +			jsonAction: 'getstatus', //CGI POST action parameter
>>> +			timeout: 5000, //XHR timeout (5s)
>>> +
>>> +			delayTimer: null, //setTimeout reference
>>> +			jqXHR: undefined, //jQuery.ajax promise reference
>>> +			get runningDelay() { //Waiting for end of delay
>>> +				return (this.delayTimer !== null);
>>> +			},
>>> +			get runningXHR() { //Waiting for CGI response
>>> +				return (this.jqXHR && (this.jqXHR.state() === 'pending'));
>>> +			},
>>> +			get isRunning() {
>>> +				return (this.runningDelay || this.runningXHR);
>>> +			}
>>> +		};
>>> +	}
>>> +
>>> +	//### Public properties ###
>>> +
>>> +	// Pakfire is running (true/false)
>>> +	set running(state) {
>>> +		if(this._states.running !== state) {
>>> +			this._states.running = state;
>>> +			this._states_onChange('running');
>>> +		}
>>> +	}
>>> +	get running() {
>>> +		return this._states.running;
>>> +	}
>>> +
>>> +	// Reboot needed (true/false)
>>> +	set reboot(state) {
>>> +		if(this._states.reboot !== state) {
>>> +			this._states.reboot = state;
>>> +			this._states_onChange('reboot');
>>> +		}
>>> +	}
>>> +	get reboot() {
>>> +		return this._states.reboot;
>>> +	}
>>> +
>>> +	// Status refresh interval in ms
>>> +	set refreshInterval(delay) {
>>> +		if(delay < 500) {
>>> +			delay = 500; //enforce reasonable minimum
>>> +		}
>>> +		this._autoRefresh.delay = delay;
>>> +	}
>>> +	get refreshInterval() {
>>> +		return this._autoRefresh.delay;
>>> +	}
>>> +
>>> +	// Document loaded (call once from jQuery.ready)
>>> +	documentReady() {
>>> +		// Status refresh late start
>>> +		if(this.running && (! this._autoRefresh.isRunning)) {
>>> +			this._autoRefresh_runNow();
>>> +		}
>>> +	}
>>> +
>>> +	//### Private properties ###
>>> +
>>> +	// Pakfire status change handler
>>> +	// property: Affected status (running, reboot, ...)
>>> +	_states_onChange(property) {
>>> +		// Always update UI
>>> +		if(this.running) {
>>> +			$('#pflog-status').text(this.i18n.get('working'));
>>> +			$('#pflog-action').empty();
>>> +		} else {
>>> +			$('#pflog-status').text(this.i18n.get('finished'));
>>> +			if(this.reboot) { //Enable return or reboot links in UI
>>> +				$('#pflog-action').html(this.i18n.get('link_reboot'));
>>> +			} else {
>>> +				$('#pflog-action').html(this.i18n.get('link_return'));
>>> +			}
>>> +		}
>>> +
>>> +		// Start/stop status refresh if Pakfire started/stopped
>>> +		if(property === 'running') {
>>> +			if(this.running) {
>>> +				this._autoRefresh_runNow();
>>> +			} else {
>>> +				this._autoRefresh_clearSchedule();
>>> +			}
>>> +		}
>>> +	}
>>> +
>>> +	//--- Status refresh scheduling functions ---
>>> +
>>> +	// Immediately perform AJAX status refresh request
>>> +	_autoRefresh_runNow() {
>>> +		if(this._autoRefresh.runningXHR) {
>>> +			return; // Don't send multiple requests
>>> +		}
>>> +		this._autoRefresh_clearSchedule(); // Stop scheduled refresh, will send immediately
>>> +
>>> +		// Send AJAX request, attach listeners
>>> +		this._autoRefresh.jqXHR = this._JSON_get(this._autoRefresh.jsonAction, this._autoRefresh.timeout);
>>> +		this._autoRefresh.jqXHR.done(function() { // Request succeeded
>>> +			if(this.running) { // Keep refreshing while Pakfire is running
>>> +				this._autoRefresh_scheduleRun();
>>> +			}
>>> +		});
>>> +		this._autoRefresh.jqXHR.fail(function() { // Request failed
>>> +			this._autoRefresh_scheduleRun(); // Try refreshing until valid status is received
>>> +		});
>>> +	}
>>> +
>>> +	// Schedule next refresh
>>> +	_autoRefresh_scheduleRun() {
>>> +		if(this._autoRefresh.runningDelay || this._autoRefresh.runningXHR) {
>>> +			return; // Refresh already scheduled or in progress
>>> +		}
>>> +		this._autoRefresh.delayTimer = window.setTimeout(function() {
>>> +			this._autoRefresh.delayTimer = null;
>>> +			this._autoRefresh_runNow();
>>> +		}.bind(this), this._autoRefresh.delay);
>>> +	}
>>> +
>>> +	// Stop scheduled refresh (can still be refreshed up to 1x if XHR is already sent)
>>> +	_autoRefresh_clearSchedule() {
>>> +		if(this._autoRefresh.runningDelay) {
>>> +			window.clearTimeout(this._autoRefresh.delayTimer);
>>> +			this._autoRefresh.delayTimer = null;
>>> +		}
>>> +	}
>>> +
>>> +	//--- JSON request & data handling ---
>>> +
>>> +	// Load JSON data from Pakfire CGI, using a POST request
>>> +	// action: POST paramter "json-[action]"
>>> +	// maxTime: XHR timeout, 0 = no timeout
>>> +	_JSON_get(action, maxTime = 0) {
>>> +		return $.ajax({
>>> +			url: '/cgi-bin/pakfire.cgi',
>>> +			method: 'POST',
>>> +			timeout: maxTime,
>>> +			context: this,
>>> +			data: {'ACTION': `json-${action}`},
>>> +			dataType: 'json' //automatically check and convert result
>>> +		})
>>> +			.done(function(response) {
>>> +				this._JSON_process(action, response);
>>> +			});
>>> +	}
>>> +
>>> +	// Process successful response from Pakfire CGI
>>> +	// action: POST paramter "json-[action]" used to send request
>>> +	// data: JSON data object
>>> +	_JSON_process(action, data) {
>>> +		// Pakfire status refresh
>>> +		if(action === this._autoRefresh.jsonAction) {
>>> +			// Update status flags
>>> +			this.running = (data['running'] != '0');
>>> +			this.reboot = (data['reboot'] != '0');
>>> +
>>> +			// Update timer display
>>> +			if(this.running && data['running_since']) {
>>> +				$('#pflog-time').text(this.i18n.get('since') + data['running_since']);
>>> +			} else {
>>> +				$('#pflog-time').empty();
>>> +			}
>>> +
>>> +			// Print log messages
>>> +			let messages = "";
>>> +			data['messages'].forEach(function(line) {
>>> +				messages += `${line}\n`;
>>> +			});
>>> +			$('#pflog-messages').text(messages);
>>> +		}
>>> +	}
>>> +}
>>> +
>>> +// Simple translation strings helper
>>> +// Format: {key: "translation"}
>>> +class PakfireI18N {
>>> +	constructor() {
>>> +		this._strings = Object.create(null); //Object without prototypes
>>> +	}
>>> +
>>> +	// Get translation
>>> +	get(key) {
>>> +		if(Object.prototype.hasOwnProperty.call(this._strings, key)) {
>>> +			return this._strings[key];
>>> +		}
>>> +		return `(undefined string '${key}')`;
>>> +	}
>>> +
>>> +	// Load key/translation object
>>> +	load(translations) {
>>> +		if(translations instanceof Object) {
>>> +			Object.assign(this._strings, translations);
>>> +		}
>>> +	}
>>> +}
>>> +
>>> +//### Initialize Pakfire ###
>>> +const pakfire = new PakfireJS();
>>> +
>>> +$(function() {
>>> +	pakfire.documentReady();
>>> +});
>>> -- 
>>> 2.27.0.windows.1