From 23d0b7ae9e23e743ca1c567ab68abb5f4c347d41 Mon Sep 17 00:00:00 2001 From: Sascha Nitsch Date: Wed, 21 Aug 2024 22:02:37 +0200 Subject: [PATCH] initital import --- .gitignore | 3 + .phan/config.php | 311 +++++++++++++++++++++++++ README.md | 111 ++++++++- composer.json | 16 ++ index.php | 20 ++ scriptedbrowser/commands/base.php | 173 ++++++++++++++ scriptedbrowser/commands/dbquery.php | 51 +++++ scriptedbrowser/control.php | 319 ++++++++++++++++++++++++++ scriptedbrowser/main.php | 331 +++++++++++++++++++++++++++ scriptedbrowser/mousebutton.php | 17 ++ scripts/01-search-google.json | 29 +++ 11 files changed, 1379 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 .phan/config.php create mode 100644 composer.json create mode 100644 index.php create mode 100644 scriptedbrowser/commands/base.php create mode 100644 scriptedbrowser/commands/dbquery.php create mode 100644 scriptedbrowser/control.php create mode 100644 scriptedbrowser/main.php create mode 100644 scriptedbrowser/mousebutton.php create mode 100644 scripts/01-search-google.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5a2d43a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +vendor +composer.lock + diff --git a/.phan/config.php b/.phan/config.php new file mode 100644 index 0000000..702b9a6 --- /dev/null +++ b/.phan/config.php @@ -0,0 +1,311 @@ + null, + + // If enabled, missing properties will be created when + // they are first seen. If false, we'll report an + // error message if there is an attempt to write + // to a class property that wasn't explicitly + // defined. + 'allow_missing_properties' => false, + + // If enabled, null can be cast as any type and any + // type can be cast to null. Setting this to true + // will cut down on false positives. + 'null_casts_as_any_type' => false, + + // If enabled, allow null to be cast as any array-like type. + // This is an incremental step in migrating away from null_casts_as_any_type. + // If null_casts_as_any_type is true, this has no effect. + 'null_casts_as_array' => false, + + // If enabled, allow any array-like type to be cast to null. + // This is an incremental step in migrating away from null_casts_as_any_type. + // If null_casts_as_any_type is true, this has no effect. + 'array_casts_as_null' => false, + + // If enabled, scalars (int, float, bool, string, null) + // are treated as if they can cast to each other. + // This does not affect checks of array keys. See scalar_array_key_cast. + 'scalar_implicit_cast' => false, + + // If enabled, any scalar array keys (int, string) + // are treated as if they can cast to each other. + // E.g. array can cast to array and vice versa. + // Normally, a scalar type such as int could only cast to/from int and mixed. + 'scalar_array_key_cast' => false, + + // If this has entries, scalars (int, float, bool, string, null) + // are allowed to perform the casts listed. + // E.g. ['int' => ['float', 'string'], 'float' => ['int'], 'string' => ['int'], 'null' => ['string']] + // allows casting null to a string, but not vice versa. + // (subset of scalar_implicit_cast) + 'scalar_implicit_partial' => [], + + // If true, seemingly undeclared variables in the global + // scope will be ignored. This is useful for projects + // with complicated cross-file globals that you have no + // hope of fixing. + 'ignore_undeclared_variables_in_global_scope' => false, + + // Set this to false to emit PhanUndeclaredFunction issues for internal functions that Phan has signatures for, + // but aren't available in the codebase, or the internal functions used to run phan + // (may lead to false positives if an extension isn't loaded) + // If this is true(default), then Phan will not warn. + 'ignore_undeclared_functions_with_known_signatures' => false, + + // Backwards Compatibility Checking. This is slow + // and expensive, but you should consider running + // it before upgrading your version of PHP to a + // new version that has backward compatibility + // breaks. + 'backward_compatibility_checks' => false, + + // If true, check to make sure the return type declared + // in the doc-block (if any) matches the return type + // declared in the method signature. + 'check_docblock_signature_return_type_match' => true, + + // (*Requires check_docblock_signature_param_type_match to be true*) + // If true, make narrowed types from phpdoc params override + // the real types from the signature, when real types exist. + // (E.g. allows specifying desired lists of subclasses, + // or to indicate a preference for non-nullable types over nullable types) + // Affects analysis of the body of the method and the param types passed in by callers. + 'prefer_narrowed_phpdoc_param_type' => true, + + // (*Requires check_docblock_signature_return_type_match to be true*) + // If true, make narrowed types from phpdoc returns override + // the real types from the signature, when real types exist. + // (E.g. allows specifying desired lists of subclasses, + // or to indicate a preference for non-nullable types over nullable types) + // Affects analysis of return statements in the body of the method and the return types passed in by callers. + 'prefer_narrowed_phpdoc_return_type' => true, + + // If enabled, check all methods that override a + // parent method to make sure its signature is + // compatible with the parent's. This check + // can add quite a bit of time to the analysis. + // This will also check if final methods are overridden, etc. + 'analyze_signature_compatibility' => true, + + // This setting maps case insensitive strings to union types. + // This is useful if a project uses phpdoc that differs from the phpdoc2 standard. + // If the corresponding value is the empty string, Phan will ignore that union type (E.g. can ignore 'the' in `@return the value`) + // If the corresponding value is not empty, Phan will act as though it saw the corresponding unionTypes(s) when the keys show up in a UnionType of @param, @return, @var, @property, etc. + // + // This matches the **entire string**, not parts of the string. + // (E.g. `@return the|null` will still look for a class with the name `the`, but `@return the` will be ignored with the below setting) + // + // (These are not aliases, this setting is ignored outside of doc comments). + // (Phan does not check if classes with these names exist) + // + // Example setting: ['unknown' => '', 'number' => 'int|float', 'char' => 'string', 'long' => 'int', 'the' => ''] + 'phpdoc_type_mapping' => [], + + // Set to true in order to attempt to detect dead + // (unreferenced) code. Keep in mind that the + // results will only be a guess given that classes, + // properties, constants and methods can be referenced + // as variables (like `$class->$property` or + // `$class->$method()`) in ways that we're unable + // to make sense of. + 'dead_code_detection' => false, + + // Set to true in order to attempt to detect unused variables. + // dead_code_detection will also enable unused variable detection. + // This has a few known false positives, e.g. for loops or branches. + 'unused_variable_detection' => true, + + // If true, this run a quick version of checks that takes less + // time at the cost of not running as thorough + // an analysis. You should consider setting this + // to true only when you wish you had more **undiagnosed** issues + // to fix in your code base. + // + // In quick-mode the scanner doesn't rescan a function + // or a method's code block every time a call is seen. + // This means that the problem here won't be detected: + // + // ```php + // false, + + // If true, then before analysis, try to simplify AST into a form + // which improves Phan's type inference in edge cases. + // + // This may conflict with 'dead_code_detection'. + // When this is true, this slows down analysis slightly. + // + // E.g. rewrites `if ($a = value() && $a > 0) {...}` + // into $a = value(); if ($a) { if ($a > 0) {...}}` + 'simplify_ast' => true, + + // Enable or disable support for generic templated + // class types. + 'generic_types_enabled' => true, + + // Override to hardcode existence and types of (non-builtin) globals in the global scope. + // Class names should be prefixed with '\\'. + // (E.g. ['_FOO' => '\\FooClass', 'page' => '\\PageClass', 'userId' => 'int']) + 'globals_type_map' => [], + + // The minimum severity level to report on. This can be + // set to Issue::SEVERITY_LOW, Issue::SEVERITY_NORMAL or + // Issue::SEVERITY_CRITICAL. Setting it to only + // critical issues is a good place to start on a big + // sloppy mature code base. + 'minimum_severity' => Issue::SEVERITY_LOW, + + // Add any issue types (such as 'PhanUndeclaredMethod') + // to this black-list to inhibit them from being reported. + 'suppress_issue_types' => ['PhanTypeInvalidDimOffset'], + + // A regular expression to match files to be excluded + // from parsing and analysis and will not be read at all. + // + // This is useful for excluding groups of test or example + // directories/files, unanalyzable files, or files that + // can't be removed for whatever reason. + // (e.g. '@Test\.php$@', or '@vendor/.*/(tests|Tests)/@') + 'exclude_file_regex' => '', +//'@^vendor/.*/(tests?|Tests?)/@', + + // A file list that defines files that will be excluded + // from parsing and analysis and will not be read at all. + // + // This is useful for excluding hopelessly unanalyzable + // files that can't be removed for whatever reason. + 'exclude_file_list' => [], + + // A directory list that defines files that will be excluded + // from static analysis, but whose class and method + // information should be included. + // + // Generally, you'll want to include the directories for + // third-party code (such as "vendor/") in this list. + // + // n.b.: If you'd like to parse but not analyze 3rd + // party code, directories containing that code + // should be added to the `directory_list` as + // to `excluce_analysis_directory_list`. + 'exclude_analysis_directory_list' => [ + 'vendor/', + ], + + // The number of processes to fork off during the analysis + // phase. + 'processes' => 1, + + // List of case-insensitive file extensions supported by Phan. + // (e.g. php, html, htm) + 'analyzed_file_extensions' => [ + 'php', + ], + + // You can put paths to stubs of internal extensions in this config option. + // If the corresponding extension is **not** loaded, then phan will use the stubs instead. + // Phan will continue using its detailed type annotations, + // but load the constants, classes, functions, and classes (and their Reflection types) + // from these stub files (doubling as valid php files). + // Use a different extension from php to avoid accidentally loading these. + // The 'tools/make_stubs' script can be used to generate your own stubs (compatible with php 7.0+ right now) + 'autoload_internal_extension_signatures' => [], + + // A list of plugin files to execute + // Plugins which are bundled with Phan can be added here by providing their name (e.g. 'AlwaysReturnPlugin') + // Alternately, you can pass in the full path to a PHP file with the plugin's implementation (e.g. 'vendor/phan/phan/.phan/plugins/AlwaysReturnPlugin.php') + 'plugins' => [ + 'AlwaysReturnPlugin', + 'DollarDollarPlugin', + 'DuplicateArrayKeyPlugin', + 'PregRegexCheckerPlugin', + 'PrintfCheckerPlugin', + 'SleepCheckerPlugin', + 'UnreachableCodePlugin', + 'NonBoolBranchPlugin', + 'NonBoolInLogicalArithPlugin', + 'InvalidVariableIssetPlugin', + 'NumericalComparisonPlugin', + 'PHPUnitNotDeadCodePlugin', + 'UnusedSuppressionPlugin', + 'UnknownElementTypePlugin', + 'DuplicateExpressionPlugin', + 'WhitespacePlugin', + 'PossiblyStaticMethodPlugin', + 'PHPDocRedundantPlugin', + ], + + // A list of directories that should be parsed for class and + // method information. After excluding the directories + // defined in exclude_analysis_directory_list, the remaining + // files will be statically analyzed for errors. + // + // Thus, both first-party and third-party code being used by + // your application should be included in this list. + 'directory_list' => [ + '.', + ], + + // A list of individual files to include in analysis + // with a path relative to the root directory of the + // project + 'file_list' => [], +]; diff --git a/README.md b/README.md index 997bd76..5a15d16 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,110 @@ -# scriptedbrowser +# Scriptedbrowser -A tool to automate a browser and create a movie recording of it \ No newline at end of file +A tool to automate a browser and create a movie recording of it. + +## Installation + +as external dependies the following tools are needed: +- a VNC server, tested with TigerVNC, others should work +- xdotool a tool to create mouse/keyboard event +- firefox with geckodriver as the browser to control +- ffmpeg to record the video + +for developing, we recommend that you install phan +by using the included composer file. +```composer install``` + +## Usage +If you start a new session, just call it via +```php index.php script_to_run.json``` +It will print out a session, that you should use for the next runs +```php index.php script_ro_tun.json session_id``` + +## Script file +A script file is a JSON file. +It contains one or more plans to execute. +Each plan has some base parameters and list of actions to do: +``` +{ + "plans": [{ + "id": "register", + "video": "register.mp4", + "mousespeed": 500, + "typespeed": 5, + "framerate": 30, + "vars": { + "baseurl": "https://contentnation.net", + "urlprefix": "${baseurl}/${lang}", + "lang": "en" + }, + "actions": [ + ] + }] +} +``` +Each plan has an id, which is an internal name, an output video file, a video frame rate and default speeds for mouse and typing. +Additionally you can define variables that can be used later in the actions block. +An example action block is given here: +``` +"actions": [ + "url ${urlprefix}/", + "sleep 2", + "startrecording", + {"name": "mousemove", "parameter": "#reglogin", "duration": 3}, + "sleep 1", + "leftclick", + "waitforurl ${urlprefix}/login.htm", + "mousemoverel 0 1", + "mousemove a[href=\"/${lang}/register.htm\"]", + "sleep 1", + "leftclick", + "waitforurl ${urlprefix}/register.htm" +] +``` +Each command can be described in 2 ways. As a string with name and parameter or as a json when more parameters should be given. + +### Valid commands +- ```injectscript URL``` injects a javascript into the current browser and page +- ```leftclick``` triggers a left mouse button click at current position +- ```mousemove cssselector``` move mouse to element with given css selector, default speed is given in block parameters, but can be overwritten by local mousespeed parameter or by given a duration in seconds to reach the goal. +- ```mousemoverel x y``` moves mouse relative to current position by x y pixels +- ```sleep time``` will sleep for given amount of seconds. +- ```startrecording``` start the recording using the given frame rate and output file in the main block. +- ```stoprecording``` finishes the recording. +- ```type text``` types in the given text +- ```url URL``` changes browser URL to given new value. This will load the page. +- ```waitforurl URL``` waits until given url is loaded, useful after a click on a link or form + +## Development and extending +## Additional commands +You can extend the tool with own commands. Each command needs to be an own file in the scriptedbrowser/commands directory. One such file is included, named dbquery.php and the matching command dbquery. + +A minimal example: +``` + $options options + * @return bool true on success + */ +function dbquery($main, $control, $options) +{ + return true; +} +``` +In the function you can do whatever your scripting needs desire. +You can also set variables to be used later by calling +```$main->setVar("name", "value")``` +and using \${name} in the script later. +Options from the script are given via $options. + +## Troubleshooting +### can't open VNC session. +Make sure you can start vnc sessions and connect to it via the normal tools. Sometimes you need to define passwords or the server does not start. +### can't record video, resolution is wrong +This depends on the vnc server, but at least for TigerVNC, create a file ```config``` in ```~/.config/tigervnc containing``` the line ```geometry=1920x1080``` diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..62430df --- /dev/null +++ b/composer.json @@ -0,0 +1,16 @@ +{ + "name": "contentnation/scriptedbrowser", + "description": "A tool to create video recordings of automated browser actions", + "type": "project", + "require-dev": { + "phan/phan": "^5.4" + }, + "license": "GPLv3+", + "authors": [ + { + "name": "Grumpydeveloper", + "email": "grumpydeveloper@contentnation.net" + } + ], + "require": {} +} diff --git a/index.php b/index.php new file mode 100644 index 0000000..1529262 --- /dev/null +++ b/index.php @@ -0,0 +1,20 @@ + [session]\n"; + return; +} +$serverHost = '127.0.0.1'; +$serverPort = 4444; +$main = new ScriptedBrowser\Main($serverHost, $serverPort, 6, $_SERVER['argc'] > 2 ? $_SERVER['argv'][2] : null); +$main->run($_SERVER['argv'][1]); diff --git a/scriptedbrowser/commands/base.php b/scriptedbrowser/commands/base.php new file mode 100644 index 0000000..ac45a5f --- /dev/null +++ b/scriptedbrowser/commands/base.php @@ -0,0 +1,173 @@ + $options options + * @return bool always true + */ +function injectjscript($main, $control, $options) +{ + $script = $main->vars($options['parameter']); + $jscript = '{const s = document.createElement("script");s.setAttribute("src", "'.$script.'"); s.setAttribute("type", "text/javascript"); document.head.append(s);}'; + $control->executeScript($jscript); + return true; +} + +/** + * do mouse left click + * + * @param \ScriptedBrowser\Main $main + * @unused-param $main + * @param \ScriptedBrowser\Control $control + * @param array $options options + * @unused-param $options + */ +function leftclick($main, $control, $options) : bool +{ + $control->click(\ScriptedBrowser\MouseButton::Left); + return true; +} + +/** + * move mouse to given element + * + * @param \ScriptedBrowser\Main $main main instance + * @param \ScriptedBrowser\Control $control control interface + * @param array $options options + * @return bool true if mouse move, false on error + */ +function mousemove($main, $control, $options) : bool +{ + $element = $control->selectCSS($main->vars($options['parameter'])); + $mouseOffset = $main->getMouseOffset(); + if ($element === []) { + return false; + } + $element = array_values($element)[0]; + $targetPos = $control->getElementRect($element); + $targetX = $targetPos['x'] + $targetPos['width'] / 2; + $targetY = $targetPos['y'] + $targetPos['height'] / 2; + // echo "move $element - $tokens[1] $targetX $targetY\n"; + $mouseX = $main->getMouseX(); + $mouseY = $main->getMouseY(); + $delta = [$targetX - $mouseX, $targetY - $mouseY]; + $framerate = $main->getFrameRate(); + $dist = sqrt($delta[0]*$delta[0] + $delta[1]*$delta[1]); + if (array_key_exists('duration', $options)) { + $steps = $options['duration'] * $framerate; + } else { + $speed = array_key_exists('mousespeed', $options) ? $options['mousespeed'] : $main->getMouseSpeed(); + $steps = $dist * $main->getFrameRate() / $speed; + } + for ($i = 0; $i < $steps; ++$i) { + $ease = \ScriptedBrowser\bezierBlend($i / $steps); + $x = $mouseX + $delta[0] * $ease; + $y = $mouseY + $delta[1] * $ease; + $control->mousemove(intval($x + $mouseOffset[0]), intval($y + $mouseOffset[1])); + usleep(1000000/$framerate); + } + $main->setMouseX($targetX); + $main->setMouseY($targetY); + $control->mousemove($targetX + $mouseOffset[0], $targetY + $mouseOffset[1]); + return true; +} + +/** + * move mouse relative + * + * @param \ScriptedBrowser\Main $main main instance + * @unused-param $main + * @param \ScriptedBrowser\Control $control control interface + * @param array $options options + * @return bool true if mouse move, false on error + */ +function mousemoverel($main, $control, $options) +{ + $coord = explode(" ", $options['parameter']); + $control->mousemoveRel(intval($coord[0], 10), intval($coord[1], 10)); + return true; +} + +/** + * sleep + * + * @param \ScriptedBrowser\Main $main + * @unused-param $main + * @param \ScriptedBrowser\Control $control control interface + * @unused-param $control + * @param array $options options + * @return bool always true + */ +function sleep($main, $control, $options) : bool +{ + usleep(intval($options['parameter']) * 1000000); + return true; +} + +/** + * type given text + * + * @param \ScriptedBrowser\Main $main + * @param \ScriptedBrowser\Control $control + * @param array $options options + * @return bool always true + */ +function type($main, $control, $options) +{ + $text = $main->vars($options['parameter']); + $speed = array_key_exists('typespeed', $options) ? $options['typespeed'] : $main->getTypeSpeed(); + $control->type($speed, $text); + return true; +} + +/** + * go to url + * + * @param \ScriptedBrowser\Main $main main instance + * @param \ScriptedBrowser\Control $control control interface + * @param array $options options + * @return bool true if url was changed, false is timeout + */ +function url($main, $control, $options) : bool +{ + $currentURL = $control->getURL(); + $url = $main->vars($options['parameter']); + if ($currentURL !== $url) { + $control->setURL($url); + } + return waitforurl($main, $control, ['parameter' => $url]); +} + +/** + * wait until given url is loaded + * + * @param \ScriptedBrowser\Main $main main instance + * @param \ScriptedBrowser\Control $control control interface + * @param array $options options + * @return boolean true if URL changed to target, false is timeout + */ +function waitforurl($main, $control, $options) : bool +{ + $target = $main->vars($options['parameter']); + $timeout = array_key_exists('timeout', $options) ? $options['timeout'] : 5; + $pollInterval = array_key_exists('pollInterval', $options) ? $options['pollInterval'] : 100; + $cancel = hrtime(true) + $timeout * 1000000000; + do { + $currentURL = $control->getURL(); + if ($currentURL === $target) { + return true; + } + usleep(1000 * $pollInterval); + } while (hrtime(true) < $cancel); + return false; +} diff --git a/scriptedbrowser/commands/dbquery.php b/scriptedbrowser/commands/dbquery.php new file mode 100644 index 0000000..7cb3030 --- /dev/null +++ b/scriptedbrowser/commands/dbquery.php @@ -0,0 +1,51 @@ + $options options + * @return bool true on success + */ +function dbquery($main, $control, $options) +{ + $dbh = new \Mysqli( + array_key_exists('hostname', $options) ? $options['hostname'] : '127.0.0.1', + array_key_exists('username', $options) ? $options['username'] : get_current_user(), + array_key_exists('password', $options) ? $options['password'] : null, + $options['database'] + ); + if ($dbh === false) { + echo "DB connection failed\n"; + return false; + } + + $sth = $dbh->query($options['query']); + if ($sth === false) { + echo "DB query failed\n"; + return false; + } + $row = $sth->fetch_array(); + if ($row === false) { + return false; + } + if ($row === false && array_key_exists('result', $options)) { + echo "DB query returned nothing"; + return false; + } + if ($row !== null && array_key_exists('result', $options)) { + $main->setVar($options['result'], $row[0]); + } + $dbh->close(); + return true; +} diff --git a/scriptedbrowser/control.php b/scriptedbrowser/control.php new file mode 100644 index 0000000..1ae48e6 --- /dev/null +++ b/scriptedbrowser/control.php @@ -0,0 +1,319 @@ +driverURL = 'http://' . $driverHost . ':' . $driverPort; + $this->driverHost = $driverHost; + $this->driverPort = $driverPort; + $this->session = ''; + $this->display = $display; + } + + /** + * click mouse button + * + * @param MouseButton $button button to press + * @return void + */ + public function click($button) + { + fwrite($this->xdotool, 'click ' . $button->value . "\n"); + } + + /** + * escape input string + * + * @param string $input input string + * @return string escaped string + */ + private static function escape(string $input): string + { + return preg_replace("/\"/", "\\\"", $input); + } + + /** + * execute a command (send to geckodriver) + * + * @param string $method HTTP method to use + * @param string $command command to send + * @param string $payload optional payload to send (JSON) + * @return mixed server response + */ + private function execute($method, $command, $payload = '{}') + { + $url = $this->driverURL; + if ($this->session !== '') { + $url .= '/session/' . $this->session; + } + if ($command !== '') { + $url .= "/$command"; + } + $ch = curl_init($url); + if ($method !== 'GET') { + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); + curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + } + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json;charset=UTF-8']); + $resp = curl_exec($ch); + curl_close($ch); + $respJson = json_decode($resp, true); + if (!is_array($respJson)) { + echo "resp: '$resp'\n"; + return []; + } + if (array_key_exists('value', $respJson)) { + return $respJson['value']; + } + print_r($respJson); + return []; + } + + /** + * execute javascript in browser + * + * @param string $script script to execute + * @return mixed result of javacript + */ + public function executeScript($script) + { + $json = json_encode(['script' => $script, 'args'=>[]]); + return $this->execute('POST', "execute/sync", $json); + } + + /** + * switch to full screen + * + * @return array{x:int,y:int,width:int,height:int} fullscreen coordinates + */ + public function fullScreen() + { + return $this->execute('POST', 'window/fullscreen'); + } + + /** + * get bounding rect of element + * + * @param string $element elementId + * @return array{x:int,y:int,width:int,height:int} coordinates of element + */ + public function getElementRect($element) + { + return $this->execute('GET', "element/$element/rect"); + } + + /** + * get current URL + * + * @return string current URL + */ + public function getURL() : string + { + return $this->execute('GET', 'url'); + } + + /** + * move mouse to absolute position + * + * @param int $x X position to move to + * @param int $y Y position to move to + * @return void + */ + public function mousemove($x, $y) + { + fwrite($this->xdotool, "mousemove $x $y\n"); + } + + /** + * move mouse relative + * @param int $x X position to move to + * @param int $y Y position to move to + * @return void + */ + public function mousemoveRel($x, $y) + { + fwrite($this->xdotool, "mousemove_relative $x $y\n"); + } + + /** + * open geckodriver if not already running + * + * @param ?string $session optional session id + * @return void + */ + public function openGeckoDriver(?string $session = null) + { + // is geckodriver running? + $socket = socket_create(AF_INET, SOCK_STREAM, SOL_TCP); + $geckosocket = @socket_connect($socket, $this->driverHost, $this->driverPort); + if ($geckosocket === false) { + // start geckodriver + echo "start geckodriver\n"; + exec("DISPLAY=:". $this->display . " nohup geckodriver 2>/dev/null >/tmp/geckodriver &"); + // wait for gecko driver + while (@socket_connect($socket, $this->driverHost, $this->driverPort) === false) { + echo "zzZZ\n"; + sleep(1); + }; + socket_close($socket); + } + + $openSession = true; + if ($session !== null) { + // todo validate session + $openSession = false; + $this->session = $session; + } + + if ($openSession) { + $sessionData = $this->execute('POST', "session", '{"capabilities" : {"alwaysMatch": {"moz:firefoxOptions":{"prefs":{"browser.display.os-zoom-behavior": 0,"layout.css.prefers-color-scheme.content-override": 1}}}, "firstMatch": [{"browserName": "firefox"}]}}'); + $this->session = $sessionData['sessionId']; + echo 'Session: '. $this->session. "\n"; + } + } + + /** + * open xdotool for mouse actions + * + * @return void + */ + public function openInput() + { + $this->xdotool = popen('DISPLAY=:' . $this->display . ' xdotool -', 'w'); + } + + /** + * start vnc session if not already running + * + * @return void + */ + public function openVNC() + { + $socket = socket_create(AF_INET, SOCK_STREAM, SOL_TCP); + $socketc = @socket_connect($socket, '127.0.0.1', 5900 + $this->display); + if ($socketc === false) { + echo "start vnc\n"; + exec('nohup vncserver :' . $this->display . ' >/dev/null 2>&1 &'); + do { + $socketc = @socket_connect($socket, '127.0.0.1', 5900 + $this->display); + if ($socketc !== false) { + break; + } + echo "zzzZZZ\n"; + sleep(1); + } while ($socket === false); + } + @socket_close($socket); + } + + /** + * find elements with given css selector + * + * @return string[] element ids + */ + public function selectCSS(string $css) + { + return $this->execute( + 'POST', + 'element', + "{\"using\": \"css selector\", \"value\": \"" . $this->escape($css) . "\"}" + ); + } + + /** + * set session id if already existing + * + * @param string $session session id + * @return void + */ + public function setSession($session) + { + $this->session = $session; + } + + /** + * set new URL + * + * @param string $url new url + * @return void + */ + public function setURL($url) + { + $this->execute('POST', 'url', "{\"url\": \"$url\"}"); + } + + /** + * type with given speed + * + * @param float $typeSpeed characters/s + * @param string $text text to type + * @return void + */ + public function type($typeSpeed, $text) + { + $delay = 1000 / $typeSpeed; + fwrite($this->xdotool, "type --delay $delay $text\n"); + } +} diff --git a/scriptedbrowser/main.php b/scriptedbrowser/main.php new file mode 100644 index 0000000..b918919 --- /dev/null +++ b/scriptedbrowser/main.php @@ -0,0 +1,331 @@ + + */ + private $vars = []; + + /** + * constructor + * + * @param string $serverHost server host name + * @param int $serverPort server port + * @param int $display VNC display id + * @param string $session existing session id from geckodriver + */ + public function __construct($serverHost, $serverPort, $display = 6, $session = null) + { + require 'commands/base.php'; + $this->display = $display; + $this->control = new Control($serverHost, $serverPort, $display); + // open vnc session + $this->control->openVNC(); + // open xdotool + $this->control->openInput(); + // open/connect to browser + $this->control->openGeckoDriver($session); + $this->control->mousemove(959, 499); + $windowpos = $this->control->fullscreen(); + $this->control->mousemove(960, 500); + $this->mouseX = 960; + $this->mouseY = 500; + $this->mouseOffsetX = $windowpos['x']; + $this->mouseOffsetY = $windowpos['y']; + } + + /** + * get frame rate + * + * @return int frame rate + */ + public function getFrameRate() + { + return $this->frameRate; + } + + /** + * get mouse offset + * + * @return array X,Y offset + */ + public function getMouseOffset() + { + return [$this->mouseOffsetX, $this->mouseOffsetY]; + } + + /** + * get mouse speed + * + * @return int speed in pixel/s + */ + public function getMouseSpeed() + { + return $this->mouseSpeed; + } + + /** + * get mouse X coordinate + * + * @return int X coordinate + */ + public function getMouseX() + { + return $this->mouseX; + } + + /** + * get mouse X coordinate + * + * @return int X coordinate + */ + public function getMouseY() + { + return $this->mouseY; + } + + /** + * get typeing speed + * + * @return int typing speed + */ + public function getTypeSpeed() + { + return $this->typeSpeed; + } + + public function run(string $instructions) : void + { + $this->control->mousemove(960, 500); + + $steps = file_get_contents($instructions); + $plans = json_decode($steps, true); + if ($plans === null) { + echo "error reading config\n"; + return; + } + $plans = $plans["plans"]; + foreach ($plans as $plan) { + echo "running " . $plan['id'] . "\n"; + $video = $plan['video']; + $framerate = $plan['framerate']; + $this->mouseSpeed = $plan['mousespeed']; + $this->typeSpeed = $plan['typespeed']; + $starttime = 0; + $this->vars = $plan['vars']; + foreach ($plan['actions'] as $action) { + if (is_string($action)) { + $tokens = explode(' ', $action, 2); + $action = [ + 'name' => $tokens[0], + 'parameter' => (sizeof($tokens) < 2) ? '' : $tokens[1] + ]; + } + $actionname = $action['name']; + if ($actionname[0] === '-') { + continue; + } + if ($starttime > 0) { + echo $actionname; + fflush(STDOUT); + } + switch ($actionname) { + case 'startrecording': + $this->ffmpeg = popen( + "ffmpeg -video_size 1920x1080 -framerate $framerate -f x11grab -i :" . $this->display . ".0 -c:v libx264 -preset ultrafast -qp 0 $video -y 2>>/dev/null > /dev/null", + "w" + ); + $starttime = hrtime(true); + echo 'start recording'; + break; + case 'stoprecording': + $this->stopRecording(); + $starttime = 0; + echo "\n"; + break; + default: + $functionName = '\\ScriptedBrowser\\Commands\\' . $actionname; + if (!function_exists($functionName)) { + $file = 'scriptedbrowser/commands/' . $actionname . '.php'; + if (file_exists($file)) { + require_once($file); + } + } + if (function_exists($functionName)) { + if (!$functionName($this, $this->control, $action)) { + echo "\r" . $action['name'] . " ". $action['parameter'] . " failed\n"; + $this->stopRecording(); + return; + } + } else { + echo "unsup " . $action['name'] . " ". array_key_exists('parameter', $action) ? $action['parameter'] : ''. "\n"; + $this->stopRecording(); + return; + } + } + if ($starttime > 0) { + $now = hrtime(true); + $d = ($now - $starttime); + $sec = $d / 1000000000; + $frame = ($sec - floor($sec)) * $framerate; + printf(" -> %d:%.0f (%.3f)\n", $sec, $frame, $sec); + } + } + } + } + /** + * set mouse X position (internal state only) + * + * @param int $mouseX new mouse X position + * @return void + */ + public function setMouseX($mouseX) + { + $this->mouseX = $mouseX; + } + + /** + * set mouse Y position (internal state only) + * + * @param int $mouseY new mouse Y position + * @return void + */ + public function setMouseY($mouseY) + { + $this->mouseY = $mouseY; + } + + /** + * programmtically set var + * + * @param string $name name to set + * @param string $value value to set + * @return void + */ + public function setVar($name, $value) + { + $this->vars[$name] = $value; + } + + /** + * stop recording + * + * @return void + */ + public function stopRecording() + { + if ($this->ffmpeg !== null) { + fwrite($this->ffmpeg, "q"); + pclose($this->ffmpeg); + } + } + + /** + * replace vars in input string + * + * @param string $input input string + * @return string input string with var replacements + */ + public function vars(string $input) : string + { + // find ${...} + while (preg_match("/\\$\\{([^\\}]+)\\}/", $input, $matches) == 1) { + $replacement = $this->vars[$matches[1]]; + $input = str_replace($matches[0], $replacement, $input); + } + return $input; + } +} diff --git a/scriptedbrowser/mousebutton.php b/scriptedbrowser/mousebutton.php new file mode 100644 index 0000000..bce7e5f --- /dev/null +++ b/scriptedbrowser/mousebutton.php @@ -0,0 +1,17 @@ +