grumpydevelop/scriptedbrowser
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

initital import

Browse source
This commit is contained in:
Sascha Nitsch 2024-08-21 22:02:37 +02:00
parent 30a1908bd7
commit 23d0b7ae9e
11 changed files with 1379 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
vendor
composer.lock

311
.phan/config.php Normal file
View file

@ -0,0 +1,311 @@
<?php
use Phan\Issue;
/**
* This configuration file was automatically generated by 'phan --init --init-level=1'
*
* TODOs (added by 'phan --init'):
*
* - Go through this file and verify that there are no missing/unnecessary files/directories.
* (E.g. this only includes direct composer dependencies - You may have to manually add indirect composer dependencies to 'directory_list')
* - Look at 'plugins' and add or remove plugins if appropriate (see https://github.com/phan/phan/tree/master/.phan/plugins#plugins)
* - Add global suppressions for pre-existing issues to suppress_issue_types (https://github.com/phan/phan/wiki/Tutorial-for-Analyzing-a-Large-Sloppy-Code-Base)
*
* This configuration will be read and overlayed on top of the
* default configuration. Command line arguments will be applied
* after this file is read.
*
* @see src/Phan/Config.php
* See Config for all configurable options.
*
* A Note About Paths
* ==================
*
* Files referenced from this file should be defined as
*
* ```
* Config::projectPath('relative_path/to/file')
* ```
*
* where the relative path is relative to the root of the
* project which is defined as either the working directory
* of the phan executable or a path passed in via the CLI
* '-d' flag.
*/
return [
// Supported values: '7.0', '7.1', '7.2', null.
// If this is set to null,
// then Phan assumes the PHP version which is closest to the minor version
// of the php executable used to execute phan.
// TODO: Choose a target_php_version for this project, or leave as null and remove this comment
'target_php_version' => null,
// If enabled, missing properties will be created when
// they are first seen. If false, we'll report an
// error message if there is an attempt to write
// to a class property that wasn't explicitly
// defined.
'allow_missing_properties' => false,
// If enabled, null can be cast as any type and any
// type can be cast to null. Setting this to true
// will cut down on false positives.
'null_casts_as_any_type' => false,
// If enabled, allow null to be cast as any array-like type.
// This is an incremental step in migrating away from null_casts_as_any_type.
// If null_casts_as_any_type is true, this has no effect.
'null_casts_as_array' => false,
// If enabled, allow any array-like type to be cast to null.
// This is an incremental step in migrating away from null_casts_as_any_type.
// If null_casts_as_any_type is true, this has no effect.
'array_casts_as_null' => false,
// If enabled, scalars (int, float, bool, string, null)
// are treated as if they can cast to each other.
// This does not affect checks of array keys. See scalar_array_key_cast.
'scalar_implicit_cast' => false,
// If enabled, any scalar array keys (int, string)
// are treated as if they can cast to each other.
// E.g. array<int,stdClass> can cast to array<string,stdClass> and vice versa.
// Normally, a scalar type such as int could only cast to/from int and mixed.
'scalar_array_key_cast' => false,
// If this has entries, scalars (int, float, bool, string, null)
// are allowed to perform the casts listed.
// E.g. ['int' => ['float', 'string'], 'float' => ['int'], 'string' => ['int'], 'null' => ['string']]
// allows casting null to a string, but not vice versa.
// (subset of scalar_implicit_cast)
'scalar_implicit_partial' => [],
// If true, seemingly undeclared variables in the global
// scope will be ignored. This is useful for projects
// with complicated cross-file globals that you have no
// hope of fixing.
'ignore_undeclared_variables_in_global_scope' => false,
// Set this to false to emit PhanUndeclaredFunction issues for internal functions that Phan has signatures for,
// but aren't available in the codebase, or the internal functions used to run phan
// (may lead to false positives if an extension isn't loaded)
// If this is true(default), then Phan will not warn.
'ignore_undeclared_functions_with_known_signatures' => false,
// Backwards Compatibility Checking. This is slow
// and expensive, but you should consider running
// it before upgrading your version of PHP to a
// new version that has backward compatibility
// breaks.
'backward_compatibility_checks' => false,
// If true, check to make sure the return type declared
// in the doc-block (if any) matches the return type
// declared in the method signature.
'check_docblock_signature_return_type_match' => true,
// (*Requires check_docblock_signature_param_type_match to be true*)
// If true, make narrowed types from phpdoc params override
// the real types from the signature, when real types exist.
// (E.g. allows specifying desired lists of subclasses,
// or to indicate a preference for non-nullable types over nullable types)
// Affects analysis of the body of the method and the param types passed in by callers.
'prefer_narrowed_phpdoc_param_type' => true,
// (*Requires check_docblock_signature_return_type_match to be true*)
// If true, make narrowed types from phpdoc returns override
// the real types from the signature, when real types exist.
// (E.g. allows specifying desired lists of subclasses,
// or to indicate a preference for non-nullable types over nullable types)
// Affects analysis of return statements in the body of the method and the return types passed in by callers.
'prefer_narrowed_phpdoc_return_type' => true,
// If enabled, check all methods that override a
// parent method to make sure its signature is
// compatible with the parent's. This check
// can add quite a bit of time to the analysis.
// This will also check if final methods are overridden, etc.
'analyze_signature_compatibility' => true,
// This setting maps case insensitive strings to union types.
// This is useful if a project uses phpdoc that differs from the phpdoc2 standard.
// If the corresponding value is the empty string, Phan will ignore that union type (E.g. can ignore 'the' in `@return the value`)
// If the corresponding value is not empty, Phan will act as though it saw the corresponding unionTypes(s) when the keys show up in a UnionType of @param, @return, @var, @property, etc.
//
// This matches the **entire string**, not parts of the string.
// (E.g. `@return the|null` will still look for a class with the name `the`, but `@return the` will be ignored with the below setting)
//
// (These are not aliases, this setting is ignored outside of doc comments).
// (Phan does not check if classes with these names exist)
//
// Example setting: ['unknown' => '', 'number' => 'int|float', 'char' => 'string', 'long' => 'int', 'the' => '']
'phpdoc_type_mapping' => [],
// Set to true in order to attempt to detect dead
// (unreferenced) code. Keep in mind that the
// results will only be a guess given that classes,
// properties, constants and methods can be referenced
// as variables (like `$class->$property` or
// `$class->$method()`) in ways that we're unable
// to make sense of.
'dead_code_detection' => false,
// Set to true in order to attempt to detect unused variables.
// dead_code_detection will also enable unused variable detection.
// This has a few known false positives, e.g. for loops or branches.
'unused_variable_detection' => true,
// If true, this run a quick version of checks that takes less
// time at the cost of not running as thorough
// an analysis. You should consider setting this
// to true only when you wish you had more **undiagnosed** issues
// to fix in your code base.
//
// In quick-mode the scanner doesn't rescan a function
// or a method's code block every time a call is seen.
// This means that the problem here won't be detected:
//
// ```php
// <?php
// function test($arg):int {
// return $arg;
// }
// test("abc");
// ```
//
// This would normally generate:
//
// ```sh
// test.php:3 TypeError return string but `test()` is declared to return int
// ```
//
// The initial scan of the function's code block has no
// type information for `$arg`. It isn't until we see
// the call and rescan test()'s code block that we can
// detect that it is actually returning the passed in
// `string` instead of an `int` as declared.
'quick_mode' => false,
// If true, then before analysis, try to simplify AST into a form
// which improves Phan's type inference in edge cases.
//
// This may conflict with 'dead_code_detection'.
// When this is true, this slows down analysis slightly.
//
// E.g. rewrites `if ($a = value() && $a > 0) {...}`
// into $a = value(); if ($a) { if ($a > 0) {...}}`
'simplify_ast' => true,
// Enable or disable support for generic templated
// class types.
'generic_types_enabled' => true,
// Override to hardcode existence and types of (non-builtin) globals in the global scope.
// Class names should be prefixed with '\\'.
// (E.g. ['_FOO' => '\\FooClass', 'page' => '\\PageClass', 'userId' => 'int'])
'globals_type_map' => [],
// The minimum severity level to report on. This can be
// set to Issue::SEVERITY_LOW, Issue::SEVERITY_NORMAL or
// Issue::SEVERITY_CRITICAL. Setting it to only
// critical issues is a good place to start on a big
// sloppy mature code base.
'minimum_severity' => Issue::SEVERITY_LOW,
// Add any issue types (such as 'PhanUndeclaredMethod')
// to this black-list to inhibit them from being reported.
'suppress_issue_types' => ['PhanTypeInvalidDimOffset'],
// A regular expression to match files to be excluded
// from parsing and analysis and will not be read at all.
//
// This is useful for excluding groups of test or example
// directories/files, unanalyzable files, or files that
// can't be removed for whatever reason.
// (e.g. '@Test\.php$@', or '@vendor/.*/(tests|Tests)/@')
'exclude_file_regex' => '',
//'@^vendor/.*/(tests?|Tests?)/@',
// A file list that defines files that will be excluded
// from parsing and analysis and will not be read at all.
//
// This is useful for excluding hopelessly unanalyzable
// files that can't be removed for whatever reason.
'exclude_file_list' => [],
// A directory list that defines files that will be excluded
// from static analysis, but whose class and method
// information should be included.
//
// Generally, you'll want to include the directories for
// third-party code (such as "vendor/") in this list.
//
// n.b.: If you'd like to parse but not analyze 3rd
// party code, directories containing that code
// should be added to the `directory_list` as
// to `excluce_analysis_directory_list`.
'exclude_analysis_directory_list' => [
'vendor/',
],
// The number of processes to fork off during the analysis
// phase.
'processes' => 1,
// List of case-insensitive file extensions supported by Phan.
// (e.g. php, html, htm)
'analyzed_file_extensions' => [
'php',
],
// You can put paths to stubs of internal extensions in this config option.
// If the corresponding extension is **not** loaded, then phan will use the stubs instead.
// Phan will continue using its detailed type annotations,
// but load the constants, classes, functions, and classes (and their Reflection types)
// from these stub files (doubling as valid php files).
// Use a different extension from php to avoid accidentally loading these.
// The 'tools/make_stubs' script can be used to generate your own stubs (compatible with php 7.0+ right now)
'autoload_internal_extension_signatures' => [],
// A list of plugin files to execute
// Plugins which are bundled with Phan can be added here by providing their name (e.g. 'AlwaysReturnPlugin')
// Alternately, you can pass in the full path to a PHP file with the plugin's implementation (e.g. 'vendor/phan/phan/.phan/plugins/AlwaysReturnPlugin.php')
'plugins' => [
'AlwaysReturnPlugin',
'DollarDollarPlugin',
'DuplicateArrayKeyPlugin',
'PregRegexCheckerPlugin',
'PrintfCheckerPlugin',
'SleepCheckerPlugin',
'UnreachableCodePlugin',
'NonBoolBranchPlugin',
'NonBoolInLogicalArithPlugin',
'InvalidVariableIssetPlugin',
'NumericalComparisonPlugin',
'PHPUnitNotDeadCodePlugin',
'UnusedSuppressionPlugin',
'UnknownElementTypePlugin',
'DuplicateExpressionPlugin',
'WhitespacePlugin',
'PossiblyStaticMethodPlugin',
'PHPDocRedundantPlugin',
],
// A list of directories that should be parsed for class and
// method information. After excluding the directories
// defined in exclude_analysis_directory_list, the remaining
// files will be statically analyzed for errors.
//
// Thus, both first-party and third-party code being used by
// your application should be included in this list.
'directory_list' => [
'.',
],
// A list of individual files to include in analysis
// with a path relative to the root directory of the
// project
'file_list' => [],
];

111
README.md
View file

@ -1,3 +1,110 @@
# scriptedbrowser # Scriptedbrowser
A tool to automate a browser and create a movie recording of it A tool to automate a browser and create a movie recording of it.
## Installation
as external dependies the following tools are needed:
- a VNC server, tested with TigerVNC, others should work
- xdotool a tool to create mouse/keyboard event
- firefox with geckodriver as the browser to control
- ffmpeg to record the video
for developing, we recommend that you install phan
by using the included composer file.
```composer install```
## Usage
If you start a new session, just call it via
```php index.php script_to_run.json```
It will print out a session, that you should use for the next runs
```php index.php script_ro_tun.json session_id```
## Script file
A script file is a JSON file.
It contains one or more plans to execute.
Each plan has some base parameters and list of actions to do:
```
{
"plans": [{
"id": "register",
"video": "register.mp4",
"mousespeed": 500,
"typespeed": 5,
"framerate": 30,
"vars": {
"baseurl": "https://contentnation.net",
"urlprefix": "${baseurl}/${lang}",
"lang": "en"
},
"actions": [
]
}]
}
```
Each plan has an id, which is an internal name, an output video file, a video frame rate and default speeds for mouse and typing.
Additionally you can define variables that can be used later in the actions block.
An example action block is given here:
```
"actions": [
"url ${urlprefix}/",
"sleep 2",
"startrecording",
{"name": "mousemove", "parameter": "#reglogin", "duration": 3},
"sleep 1",
"leftclick",
"waitforurl ${urlprefix}/login.htm",
"mousemoverel 0 1",
"mousemove a[href=\"/${lang}/register.htm\"]",
"sleep 1",
"leftclick",
"waitforurl ${urlprefix}/register.htm"
]
```
Each command can be described in 2 ways. As a string with name and parameter or as a json when more parameters should be given.
### Valid commands
- ```injectscript URL``` injects a javascript into the current browser and page
- ```leftclick``` triggers a left mouse button click at current position
- ```mousemove cssselector``` move mouse to element with given css selector, default speed is given in block parameters, but can be overwritten by local mousespeed parameter or by given a duration in seconds to reach the goal.
- ```mousemoverel x y``` moves mouse relative to current position by x y pixels
- ```sleep time``` will sleep for given amount of seconds.
- ```startrecording``` start the recording using the given frame rate and output file in the main block.
- ```stoprecording``` finishes the recording.
- ```type text``` types in the given text
- ```url URL``` changes browser URL to given new value. This will load the page.
- ```waitforurl URL``` waits until given url is loaded, useful after a click on a link or form
## Development and extending
## Additional commands
You can extend the tool with own commands. Each command needs to be an own file in the scriptedbrowser/commands directory. One such file is included, named dbquery.php and the matching command dbquery.
A minimal example:
```
<?php
namespace ScriptedBrowser\Commands;
/**
* inject and execute given javascript
*
* @param \ScriptedBrowser\Main $main main instance
* @param \ScriptedBrowser\Control $control control interface
* @param array<string,mixed> $options options
* @return bool true on success
*/
function dbquery($main, $control, $options)
{
return true;
}
```
In the function you can do whatever your scripting needs desire.
You can also set variables to be used later by calling
```$main->setVar("name", "value")```
and using \${name} in the script later.
Options from the script are given via $options.
## Troubleshooting
### can't open VNC session.
Make sure you can start vnc sessions and connect to it via the normal tools. Sometimes you need to define passwords or the server does not start.
### can't record video, resolution is wrong
This depends on the vnc server, but at least for TigerVNC, create a file ```config``` in ```~/.config/tigervnc containing``` the line ```geometry=1920x1080```

16
composer.json Normal file
View file

@ -0,0 +1,16 @@
{
"name": "contentnation/scriptedbrowser",
"description": "A tool to create video recordings of automated browser actions",
"type": "project",
"require-dev": {
"phan/phan": "^5.4"
},
"license": "GPLv3+",
"authors": [
{
"name": "Grumpydeveloper",
"email": "grumpydeveloper@contentnation.net"
}
],
"require": {}
}

20
index.php Normal file
View file

@ -0,0 +1,20 @@
<?php
/**
* SPDX-FileCopyrightText: 2024 Sascha Nitsch (grumpydeveloper) https://contentnation.net/@grumpydevelop
* SPDX-License-Identifier: GPL-3.0-or-later
*
* @author Sascha Nitsch (grumpydeveloper)
**/
spl_autoload_register(static function (string $className) {
include str_replace("\\", "/", strtolower($className)) . '.php';
});
if ($_SERVER['argc'] < 2) {
echo "usage php ".$_SERVER['argv'][0]." <scriptfile> [session]\n";
return;
}
$serverHost = '127.0.0.1';
$serverPort = 4444;
$main = new ScriptedBrowser\Main($serverHost, $serverPort, 6, $_SERVER['argc'] > 2 ? $_SERVER['argv'][2] : null);
$main->run($_SERVER['argv'][1]);

173
scriptedbrowser/commands/base.php Normal file
View file

@ -0,0 +1,173 @@
<?php
/**
* SPDX-FileCopyrightText: 2024 Sascha Nitsch (grumpydeveloper) https://contentnation.net/@grumpydevelop
* SPDX-License-Identifier: GPL-3.0-or-later
*
* @author Sascha Nitsch (grumpydeveloper)
**/
namespace ScriptedBrowser\Commands;
/**
* inject and execute given javascript
*
* @param \ScriptedBrowser\Main $main main instance
* @param \ScriptedBrowser\Control $control control interface
* @param array<string,mixed> $options options
* @return bool always true
*/
function injectjscript($main, $control, $options)
{
$script = $main->vars($options['parameter']);
$jscript = '{const s = document.createElement("script");s.setAttribute("src", "'.$script.'"); s.setAttribute("type", "text/javascript"); document.head.append(s);}';
$control->executeScript($jscript);
return true;
}
/**
* do mouse left click
*
* @param \ScriptedBrowser\Main $main
* @unused-param $main
* @param \ScriptedBrowser\Control $control
* @param array<string,mixed> $options options
* @unused-param $options
*/
function leftclick($main, $control, $options) : bool
{
$control->click(\ScriptedBrowser\MouseButton::Left);
return true;
}
/**
* move mouse to given element
*
* @param \ScriptedBrowser\Main $main main instance
* @param \ScriptedBrowser\Control $control control interface
* @param array<string,mixed> $options options
* @return bool true if mouse move, false on error
*/
function mousemove($main, $control, $options) : bool
{
$element = $control->selectCSS($main->vars($options['parameter']));
$mouseOffset = $main->getMouseOffset();
if ($element === []) {
return false;
}
$element = array_values($element)[0];
$targetPos = $control->getElementRect($element);
$targetX = $targetPos['x'] + $targetPos['width'] / 2;
$targetY = $targetPos['y'] + $targetPos['height'] / 2;
// echo "move $element - $tokens[1] $targetX $targetY\n";
$mouseX = $main->getMouseX();
$mouseY = $main->getMouseY();
$delta = [$targetX - $mouseX, $targetY - $mouseY];
$framerate = $main->getFrameRate();
$dist = sqrt($delta[0]*$delta[0] + $delta[1]*$delta[1]);
if (array_key_exists('duration', $options)) {
$steps = $options['duration'] * $framerate;
} else {
$speed = array_key_exists('mousespeed', $options) ? $options['mousespeed'] : $main->getMouseSpeed();
$steps = $dist * $main->getFrameRate() / $speed;
}
for ($i = 0; $i < $steps; ++$i) {
$ease = \ScriptedBrowser\bezierBlend($i / $steps);
$x = $mouseX + $delta[0] * $ease;
$y = $mouseY + $delta[1] * $ease;
$control->mousemove(intval($x + $mouseOffset[0]), intval($y + $mouseOffset[1]));
usleep(1000000/$framerate);
}
$main->setMouseX($targetX);
$main->setMouseY($targetY);
$control->mousemove($targetX + $mouseOffset[0], $targetY + $mouseOffset[1]);
return true;
}
/**
* move mouse relative
*
* @param \ScriptedBrowser\Main $main main instance
* @unused-param $main
* @param \ScriptedBrowser\Control $control control interface
* @param array<string,mixed> $options options
* @return bool true if mouse move, false on error
*/
function mousemoverel($main, $control, $options)
{
$coord = explode(" ", $options['parameter']);
$control->mousemoveRel(intval($coord[0], 10), intval($coord[1], 10));
return true;
}
/**
* sleep
*
* @param \ScriptedBrowser\Main $main
* @unused-param $main
* @param \ScriptedBrowser\Control $control control interface
* @unused-param $control
* @param array<string,mixed> $options options
* @return bool always true
*/
function sleep($main, $control, $options) : bool
{
usleep(intval($options['parameter']) * 1000000);
return true;
}
/**
* type given text
*
* @param \ScriptedBrowser\Main $main
* @param \ScriptedBrowser\Control $control
* @param array<string,mixed> $options options
* @return bool always true
*/
function type($main, $control, $options)
{
$text = $main->vars($options['parameter']);
$speed = array_key_exists('typespeed', $options) ? $options['typespeed'] : $main->getTypeSpeed();
$control->type($speed, $text);
return true;
}
/**
* go to url
*
* @param \ScriptedBrowser\Main $main main instance
* @param \ScriptedBrowser\Control $control control interface
* @param array<string,mixed> $options options
* @return bool true if url was changed, false is timeout
*/
function url($main, $control, $options) : bool
{
$currentURL = $control->getURL();
$url = $main->vars($options['parameter']);
if ($currentURL !== $url) {
$control->setURL($url);
}
return waitforurl($main, $control, ['parameter' => $url]);
}
/**
* wait until given url is loaded
*
* @param \ScriptedBrowser\Main $main main instance
* @param \ScriptedBrowser\Control $control control interface
* @param array<string,mixed> $options options
* @return boolean true if URL changed to target, false is timeout
*/
function waitforurl($main, $control, $options) : bool
{
$target = $main->vars($options['parameter']);
$timeout = array_key_exists('timeout', $options) ? $options['timeout'] : 5;
$pollInterval = array_key_exists('pollInterval', $options) ? $options['pollInterval'] : 100;
$cancel = hrtime(true) + $timeout * 1000000000;
do {
$currentURL = $control->getURL();
if ($currentURL === $target) {
return true;
}
usleep(1000 * $pollInterval);
} while (hrtime(true) < $cancel);
return false;
}

51
scriptedbrowser/commands/dbquery.php Normal file
View file

@ -0,0 +1,51 @@
<?php
/**
* SPDX-FileCopyrightText: 2024 Sascha Nitsch (grumpydeveloper) https://contentnation.net/@grumpydevelop
* SPDX-License-Identifier: GPL-3.0-or-later
*
* @author Sascha Nitsch (grumpydeveloper)
**/
namespace ScriptedBrowser\Commands;
/**
* inject and execute given javascript
*
* @param \ScriptedBrowser\Main $main main instance
* @param \ScriptedBrowser\Control $control control interface
* @unused-param $control
* @param array<string,mixed> $options options
* @return bool true on success
*/
function dbquery($main, $control, $options)
{
$dbh = new \Mysqli(
array_key_exists('hostname', $options) ? $options['hostname'] : '127.0.0.1',
array_key_exists('username', $options) ? $options['username'] : get_current_user(),
array_key_exists('password', $options) ? $options['password'] : null,
$options['database']
);
if ($dbh === false) {
echo "DB connection failed\n";
return false;
}
$sth = $dbh->query($options['query']);
if ($sth === false) {
echo "DB query failed\n";
return false;
}
$row = $sth->fetch_array();
if ($row === false) {
return false;
}
if ($row === false && array_key_exists('result', $options)) {
echo "DB query returned nothing";
return false;
}
if ($row !== null && array_key_exists('result', $options)) {
$main->setVar($options['result'], $row[0]);
}
$dbh->close();
return true;
}

319
scriptedbrowser/control.php Normal file
View file

@ -0,0 +1,319 @@
<?php
/**
* SPDX-FileCopyrightText: 2024 Sascha Nitsch (grumpydeveloper) https://contentnation.net/@grumpydevelop
* SPDX-License-Identifier: GPL-3.0-or-later
*
* @author Sascha Nitsch (grumpydeveloper)
**/
namespace ScriptedBrowser;
/**
* control function for gecko driver
*/
class Control
{
/**
* display id
*
* @var int
*/
private $display;
/**
* geckodriver host
*
* @var string
*/
private $driverHost;
/**
* geckodriver port
*
* @var int
*/
private $driverPort;
/**
* geckodriver url
*
* @var string $serverURL
*/
private $driverURL;
/**
* session ID
*
* @var string
*/
private $session;
/**
* xdo resource
*
* @var resource
*/
private $xdotool = null;
/**
* constructor
*
* @param string $driverHost hostname of geckodriver
* @param int $driverPort port of geckodriver
* @param int $display display id of vnc
*/
public function __construct($driverHost, $driverPort, $display)
{
$this->driverURL = 'http://' . $driverHost . ':' . $driverPort;
$this->driverHost = $driverHost;
$this->driverPort = $driverPort;
$this->session = '';
$this->display = $display;
}
/**
* click mouse button
*
* @param MouseButton $button button to press
* @return void
*/
public function click($button)
{
fwrite($this->xdotool, 'click ' . $button->value . "\n");
}
/**
* escape input string
*
* @param string $input input string
* @return string escaped string
*/
private static function escape(string $input): string
{
return preg_replace("/\"/", "\\\"", $input);
}
/**
* execute a command (send to geckodriver)
*
* @param string $method HTTP method to use
* @param string $command command to send
* @param string $payload optional payload to send (JSON)
* @return mixed server response
*/
private function execute($method, $command, $payload = '{}')
{
$url = $this->driverURL;
if ($this->session !== '') {
$url .= '/session/' . $this->session;
}
if ($command !== '') {
$url .= "/$command";
}
$ch = curl_init($url);
if ($method !== 'GET') {
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method);
curl_setopt($ch, CURLOPT_POSTFIELDS, $payload);
}
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json;charset=UTF-8']);
$resp = curl_exec($ch);
curl_close($ch);
$respJson = json_decode($resp, true);
if (!is_array($respJson)) {
echo "resp: '$resp'\n";
return [];
}
if (array_key_exists('value', $respJson)) {
return $respJson['value'];
}
print_r($respJson);
return [];
}
/**
* execute javascript in browser
*
* @param string $script script to execute
* @return mixed result of javacript
*/
public function executeScript($script)
{
$json = json_encode(['script' => $script, 'args'=>[]]);
return $this->execute('POST', "execute/sync", $json);
}
/**
* switch to full screen
*
* @return array{x:int,y:int,width:int,height:int} fullscreen coordinates
*/
public function fullScreen()
{
return $this->execute('POST', 'window/fullscreen');
}
/**
* get bounding rect of element
*
* @param string $element elementId
* @return array{x:int,y:int,width:int,height:int} coordinates of element
*/
public function getElementRect($element)
{
return $this->execute('GET', "element/$element/rect");
}
/**
* get current URL
*
* @return string current URL
*/
public function getURL() : string
{
return $this->execute('GET', 'url');
}
/**
* move mouse to absolute position
*
* @param int $x X position to move to
* @param int $y Y position to move to
* @return void
*/
public function mousemove($x, $y)
{
fwrite($this->xdotool, "mousemove $x $y\n");
}
/**
* move mouse relative
* @param int $x X position to move to
* @param int $y Y position to move to
* @return void
*/
public function mousemoveRel($x, $y)
{
fwrite($this->xdotool, "mousemove_relative $x $y\n");
}
/**
* open geckodriver if not already running
*
* @param ?string $session optional session id
* @return void
*/
public function openGeckoDriver(?string $session = null)
{
// is geckodriver running?
$socket = socket_create(AF_INET, SOCK_STREAM, SOL_TCP);
$geckosocket = @socket_connect($socket, $this->driverHost, $this->driverPort);
if ($geckosocket === false) {
// start geckodriver
echo "start geckodriver\n";
exec("DISPLAY=:". $this->display . " nohup geckodriver 2>/dev/null >/tmp/geckodriver &");
// wait for gecko driver
while (@socket_connect($socket, $this->driverHost, $this->driverPort) === false) {
echo "zzZZ\n";
sleep(1);
};
socket_close($socket);
}
$openSession = true;
if ($session !== null) {
// todo validate session
$openSession = false;
$this->session = $session;
}
if ($openSession) {
$sessionData = $this->execute('POST', "session", '{"capabilities" : {"alwaysMatch": {"moz:firefoxOptions":{"prefs":{"browser.display.os-zoom-behavior": 0,"layout.css.prefers-color-scheme.content-override": 1}}}, "firstMatch": [{"browserName": "firefox"}]}}');
$this->session = $sessionData['sessionId'];
echo 'Session: '. $this->session. "\n";
}
}
/**
* open xdotool for mouse actions
*
* @return void
*/
public function openInput()
{
$this->xdotool = popen('DISPLAY=:' . $this->display . ' xdotool -', 'w');
}
/**
* start vnc session if not already running
*
* @return void
*/
public function openVNC()
{
$socket = socket_create(AF_INET, SOCK_STREAM, SOL_TCP);
$socketc = @socket_connect($socket, '127.0.0.1', 5900 + $this->display);
if ($socketc === false) {
echo "start vnc\n";
exec('nohup vncserver :' . $this->display . ' >/dev/null 2>&1 &');
do {
$socketc = @socket_connect($socket, '127.0.0.1', 5900 + $this->display);
if ($socketc !== false) {
break;
}
echo "zzzZZZ\n";
sleep(1);
} while ($socket === false);
}
@socket_close($socket);
}
/**
* find elements with given css selector
*
* @return string[] element ids
*/
public function selectCSS(string $css)
{
return $this->execute(
'POST',
'element',
"{\"using\": \"css selector\", \"value\": \"" . $this->escape($css) . "\"}"
);
}
/**
* set session id if already existing
*
* @param string $session session id
* @return void
*/
public function setSession($session)
{
$this->session = $session;
}
/**
* set new URL
*
* @param string $url new url
* @return void
*/
public function setURL($url)
{
$this->execute('POST', 'url', "{\"url\": \"$url\"}");
}
/**
* type with given speed
*
* @param float $typeSpeed characters/s
* @param string $text text to type
* @return void
*/
public function type($typeSpeed, $text)
{
$delay = 1000 / $typeSpeed;
fwrite($this->xdotool, "type --delay $delay $text\n");
}
}

331
scriptedbrowser/main.php Normal file
View file

@ -0,0 +1,331 @@
<?php
/**
* SPDX-FileCopyrightText: 2024 Sascha Nitsch (grumpydeveloper) https://contentnation.net/@grumpydevelop
* SPDX-License-Identifier: GPL-3.0-or-later
*
* @author Sascha Nitsch (grumpydeveloper)
**/
namespace ScriptedBrowser;
/**
* Calculate bezier blending from 0...1
*
* @param float $t input value 0...1
* @return float
*/
function bezierBlend(float $t)
{
return $t * $t * (3.0 - 2.0 * $t);
}
/**
* main class
*/
class Main
{
/**
* control instance
*
* @var Control
*/
private $control;
/**
* display id
* @var int $display
*/
private $display;
/**
* recording process
* @var ?resource
*/
private $ffmpeg = null;
/**
* frame rate of recording
*
* @var int $frameRate
*/
private $frameRate = 30;
/**
* X mouse offset of browser to left edge
*
* @var int $mouseOffsetX
*/
private $mouseOffsetX = 0;
/**
* Y mouse offset of browser to top edge
*
* @var int $mouseOffsetY
*/
private $mouseOffsetY = 0;
/**
* mouse movement speed (pixel per second)
*
* @var int $mouseSpeed
*/
private $mouseSpeed = 500;
/**
* mouse position X
*
* @var int $mouseX
*/
private $mouseX = 0;
/**
* mouse position Y
*
* @var int $mouseY
*/
private $mouseY = 0;
/**
* typing speed in characters/s
*
* @var int $typeSpeed
*/
private $typeSpeed = 5;
/**
* var storage
*
* @var array<string, string>
*/
private $vars = [];
/**
* constructor
*
* @param string $serverHost server host name
* @param int $serverPort server port
* @param int $display VNC display id
* @param string $session existing session id from geckodriver
*/
public function __construct($serverHost, $serverPort, $display = 6, $session = null)
{
require 'commands/base.php';
$this->display = $display;
$this->control = new Control($serverHost, $serverPort, $display);
// open vnc session
$this->control->openVNC();
// open xdotool
$this->control->openInput();
// open/connect to browser
$this->control->openGeckoDriver($session);
$this->control->mousemove(959, 499);
$windowpos = $this->control->fullscreen();
$this->control->mousemove(960, 500);
$this->mouseX = 960;
$this->mouseY = 500;
$this->mouseOffsetX = $windowpos['x'];
$this->mouseOffsetY = $windowpos['y'];
}
/**
* get frame rate
*
* @return int frame rate
*/
public function getFrameRate()
{
return $this->frameRate;
}
/**
* get mouse offset
*
* @return array<int> X,Y offset
*/
public function getMouseOffset()
{
return [$this->mouseOffsetX, $this->mouseOffsetY];
}
/**
* get mouse speed
*
* @return int speed in pixel/s
*/
public function getMouseSpeed()
{
return $this->mouseSpeed;
}
/**
* get mouse X coordinate
*
* @return int X coordinate
*/
public function getMouseX()
{
return $this->mouseX;
}
/**
* get mouse X coordinate
*
* @return int X coordinate
*/
public function getMouseY()
{
return $this->mouseY;
}
/**
* get typeing speed
*
* @return int typing speed
*/
public function getTypeSpeed()
{
return $this->typeSpeed;
}
public function run(string $instructions) : void
{
$this->control->mousemove(960, 500);
$steps = file_get_contents($instructions);
$plans = json_decode($steps, true);
if ($plans === null) {
echo "error reading config\n";
return;
}
$plans = $plans["plans"];
foreach ($plans as $plan) {
echo "running " . $plan['id'] . "\n";
$video = $plan['video'];
$framerate = $plan['framerate'];
$this->mouseSpeed = $plan['mousespeed'];
$this->typeSpeed = $plan['typespeed'];
$starttime = 0;
$this->vars = $plan['vars'];
foreach ($plan['actions'] as $action) {
if (is_string($action)) {
$tokens = explode(' ', $action, 2);
$action = [
'name' => $tokens[0],
'parameter' => (sizeof($tokens) < 2) ? '' : $tokens[1]
];
}
$actionname = $action['name'];
if ($actionname[0] === '-') {
continue;
}
if ($starttime > 0) {
echo $actionname;
fflush(STDOUT);
}
switch ($actionname) {
case 'startrecording':
$this->ffmpeg = popen(
"ffmpeg -video_size 1920x1080 -framerate $framerate -f x11grab -i :" . $this->display . ".0 -c:v libx264 -preset ultrafast -qp 0 $video -y 2>>/dev/null > /dev/null",
"w"
);
$starttime = hrtime(true);
echo 'start recording';
break;
case 'stoprecording':
$this->stopRecording();
$starttime = 0;
echo "\n";
break;
default:
$functionName = '\\ScriptedBrowser\\Commands\\' . $actionname;
if (!function_exists($functionName)) {
$file = 'scriptedbrowser/commands/' . $actionname . '.php';
if (file_exists($file)) {
require_once($file);
}
}
if (function_exists($functionName)) {
if (!$functionName($this, $this->control, $action)) {
echo "\r" . $action['name'] . " ". $action['parameter'] . " failed\n";
$this->stopRecording();
return;
}
} else {
echo "unsup " . $action['name'] . " ". array_key_exists('parameter', $action) ? $action['parameter'] : ''. "\n";
$this->stopRecording();
return;
}
}
if ($starttime > 0) {
$now = hrtime(true);
$d = ($now - $starttime);
$sec = $d / 1000000000;
$frame = ($sec - floor($sec)) * $framerate;
printf(" -> %d:%.0f (%.3f)\n", $sec, $frame, $sec);
}
}
}
}
/**
* set mouse X position (internal state only)
*
* @param int $mouseX new mouse X position
* @return void
*/
public function setMouseX($mouseX)
{
$this->mouseX = $mouseX;
}
/**
* set mouse Y position (internal state only)
*
* @param int $mouseY new mouse Y position
* @return void
*/
public function setMouseY($mouseY)
{
$this->mouseY = $mouseY;
}
/**
* programmtically set var
*
* @param string $name name to set
* @param string $value value to set
* @return void
*/
public function setVar($name, $value)
{
$this->vars[$name] = $value;
}
/**
* stop recording
*
* @return void
*/
public function stopRecording()
{
if ($this->ffmpeg !== null) {
fwrite($this->ffmpeg, "q");
pclose($this->ffmpeg);
}
}
/**
* replace vars in input string
*
* @param string $input input string
* @return string input string with var replacements
*/
public function vars(string $input) : string
{
// find ${...}
while (preg_match("/\\$\\{([^\\}]+)\\}/", $input, $matches) == 1) {
$replacement = $this->vars[$matches[1]];
$input = str_replace($matches[0], $replacement, $input);
}
return $input;
}
}

17
scriptedbrowser/mousebutton.php Normal file
View file

@ -0,0 +1,17 @@
<?php
/**
* SPDX-FileCopyrightText: 2024 Sascha Nitsch (grumpydeveloper) https://contentnation.net/@grumpydevelop
* SPDX-License-Identifier: GPL-3.0-or-later
*
* @author Sascha Nitsch (grumpydeveloper)
**/
namespace ScriptedBrowser;
/**
* list of mouse buttons
*/
enum MouseButton: int
{
case Left = 1;
}

29
scripts/01-search-google.json Normal file
View file

@ -0,0 +1,29 @@
{
"plans": [{
"id": "search",
"video": "search.mp4",
"mousespeed": 500,
"typespeed": 5,
"framerate": 30,
"vars": {
},
"actions": [
"url https://www.google.com/",
"sleep 2",
"startrecording",
{"name": "mousemove", "parameter": "#W0wltc", "duration": 3},
"leftclick",
{"name": "mousemove", "parameter": "textarea:not(.csi)", "duration": 3},
"sleep 1",
"leftclick",
"type contentnation",
"leftclick",
"sleep 1",
"mousemove input[type=\"submit\"]",
"sleep 1",
"leftclick",
"sleep 5",
"stoprecording"
]
}]
}