270 lines
8.3 KiB
Perl
Executable File
270 lines
8.3 KiB
Perl
Executable File
#!/bin/perl
|
|
|
|
use strict;
|
|
use warnings;
|
|
use feature ("signatures");
|
|
use JSON;
|
|
use WWW::Curl::Easy;
|
|
use File::Path("make_path");
|
|
|
|
use Data::Dumper;
|
|
$Data::Dumper::Pair = " : ";
|
|
$Data::Dumper::Indent = 2;
|
|
|
|
use YAML;
|
|
$YAML::Preserve = 1;
|
|
|
|
# TODO: Show hook/clone output in a prettier fashion (like docker buildx)
|
|
# TODO: Allow branch selection
|
|
# TODO: Allow updating a single repo
|
|
# TODO: Add flags to allow checking for unclean trees, only cloning, only pulling
|
|
# TODO: Check if directories are empty before cloning any repos to allow cloning in any order
|
|
|
|
use constant USERAGENT => "User-Agent: MarxBot/4.2.0 (A script reading some information about repos)";
|
|
use constant URL_REGEX =>
|
|
qr/^((.*(?:@|:\/\/))[a-zA-Z0-9-_.]+(?:\/|:)(?:[a-zA-Z0-9-_.\/]+\/)?([a-zA-Z0-9-_.]+?)\/([a-zA-Z0-9-_.]+?)(?:\.git)?\/?)$/;
|
|
|
|
my @handles;
|
|
my @messages;
|
|
my %conf;
|
|
my $active_repos = 0;
|
|
my $active_requests = 0;
|
|
|
|
|
|
sub info($message) {
|
|
print(`tput setaf 4; tput bold; echo '$message'; tput sgr 0`);
|
|
}
|
|
|
|
|
|
sub error($message) {
|
|
push(@messages, `tput setaf 1; tput bold; echo '$message'; tput sgr 0`);
|
|
}
|
|
|
|
|
|
sub warning($message) {
|
|
push(@messages, `tput setaf 3; tput bold; echo '$message'; tput sgr 0`);
|
|
}
|
|
|
|
|
|
sub set_curl($handle, $url, @headers) {
|
|
$handles[$handle]{curl} = WWW::Curl::Easy->new;
|
|
$handles[$handle]{curl}->setopt(CURLOPT_URL, $url);
|
|
$handles[$handle]{curl}->pushopt(CURLOPT_HTTPHEADER, [USERAGENT]);
|
|
for my $header (@headers) {
|
|
$handles[$handle]{curl}->pushopt(CURLOPT_HTTPHEADER, [$header]);
|
|
}
|
|
$handles[$handle]{curl}->setopt(CURLOPT_PRIVATE, $handle);
|
|
$handles[$handle]{curl}->setopt(CURLOPT_WRITEDATA, \$handles[$handle]{memory});
|
|
}
|
|
|
|
|
|
sub add_callback($handle, $callback) {
|
|
push(@{$handles[$handle]{callbacks}}, $callback);
|
|
}
|
|
|
|
|
|
sub exec_curl($handle) {
|
|
my $curl = $handles[$handle]{curl};
|
|
$curl->perform;
|
|
my $status = $curl->getinfo(CURLINFO_HTTP_CODE);
|
|
if ($status < 200 || $status > 300) {
|
|
my $url = $curl->getinfo(CURLOPT_URL);
|
|
error("Curl on $url failed with code $status");
|
|
}
|
|
}
|
|
|
|
|
|
sub json_decode($handle) {
|
|
$handles[$handle]{memory} = JSON::decode_json($handles[$handle]{memory});
|
|
}
|
|
|
|
|
|
sub url_filter($handle) {
|
|
my @tmp;
|
|
my $lookup = $conf{lookups}[$handles[$handle]->{lookup}];
|
|
for my $repo (@{$handles[$handle]{memory}}) {
|
|
if ($repo->{$lookup->{url_field}}) {
|
|
push(@tmp, $repo->{$lookup->{url_field}});
|
|
} else {
|
|
error("Failed to extract $lookup->{url_field} while processing lookup: $lookup->{name}");
|
|
}
|
|
}
|
|
$handles[$handle]{memory} = \@tmp;
|
|
}
|
|
|
|
|
|
sub inject_conf_urls($handle) {
|
|
for my $url (@{$conf{extra_urls}}) {
|
|
push(@{$handles[$handle]{memory}}, $url);
|
|
}
|
|
}
|
|
|
|
|
|
sub process_urls($handle) {
|
|
my @tmp;
|
|
for my $url (@{$handles[$handle]{memory}}) {
|
|
my %repo;
|
|
if ($url =~ URL_REGEX) {
|
|
$repo{url} = $1;
|
|
$repo{owner} = $3;
|
|
$repo{name} = $4;
|
|
$repo{fullname} = "$3/$4";
|
|
|
|
if (substr($2, -1) eq "@") {
|
|
$repo{protocol} = "ssh";
|
|
} else {
|
|
$repo{protocol} = $2;
|
|
}
|
|
} else {
|
|
error("Failed to parse url: $url");
|
|
next;
|
|
}
|
|
|
|
next if (grep($_ eq $repo{fullname}, @{$conf{skip_repos}}));
|
|
|
|
my $path;
|
|
|
|
my $repodir = $repo{name};
|
|
for my $pt (@{$conf{path_transformations}}) {
|
|
if ($repo{fullname} =~ qr/$pt->{match}/) {
|
|
$repodir =~ s/$pt->{replace}/$pt->{with}/;
|
|
}
|
|
}
|
|
|
|
for my $directory (@{$conf{directories}}) {
|
|
for my $regex (@{$directory->{repos}}) {
|
|
if (
|
|
$repo{fullname} =~ qr/^$regex$/
|
|
&& (!$directory->{lookups}
|
|
|| grep(qr/$conf{lookups}[ $handles[$handle]{lookup} ]{name}/, @{$directory->{lookups}}))
|
|
) {
|
|
$repo{path} = `printf $directory->{path}/$repodir`;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!$repo{path} && !$conf{lookups}[$handles[$handle]{lookup}]{block_unsorted}) {
|
|
$repo{path} = `printf $conf{unsorted_directory}/$repodir`;
|
|
} elsif ($repo{path}) {
|
|
} else {
|
|
next;
|
|
}
|
|
|
|
my $clone_hook = "$conf{hook_dir}/clone/$repo{owner}:$repo{name}";
|
|
my $pull_hook = "$conf{hook_dir}/pull/$repo{owner}:$repo{name}";
|
|
(-x $clone_hook) and $repo{clone_hook} = "cd $repo{path} && $clone_hook";
|
|
(-x $pull_hook) and $repo{pull_hook} = "cd $repo{path} && $pull_hook";
|
|
push(@tmp, \%repo);
|
|
}
|
|
$handles[$handle]{memory} = \@tmp;
|
|
}
|
|
|
|
|
|
sub dump_mem($handle) {
|
|
print("------ Handle $handle ------\n");
|
|
print Dumper($handles[$handle]->{memory});
|
|
}
|
|
|
|
|
|
sub folder_is_empty($directory) {
|
|
opendir(my $dh, $directory) or return 1;
|
|
return scalar(grep($_ ne "." && $_ ne "..", readdir($dh))) == 0;
|
|
}
|
|
|
|
|
|
sub handle_repos($handle) {
|
|
for my $repo (@{$handles[$handle]->{memory}}) {
|
|
if (folder_is_empty("$repo->{path}")) {
|
|
make_path($repo->{path});
|
|
info("Cloning $repo->{fullname}");
|
|
`git -C '$repo->{path}' clone $conf{clone_flags} '$repo->{url}' .`;
|
|
($? != 0) and error("Failed to clone $repo->{url} to $repo->{path}");
|
|
if ($repo->{clone_hook}) {
|
|
info("Running clone hook for $repo->{fullname}");
|
|
`$repo->{clone_hook}`;
|
|
}
|
|
($? != 0) and error("Failed to execute clone hook for $repo->{fullname}");
|
|
} elsif (!folder_is_empty("$repo->{path}/.git")) {
|
|
info("Pulling $repo->{fullname} to $repo->{path}");
|
|
if (`git -C $repo->{path} status -z`) {
|
|
warning("$repo->{path} has an unclean tree.");
|
|
} else {
|
|
`git -C $repo->{path} pull $conf{pull_flags}`;
|
|
}
|
|
($? != 0) and error("Failed to pull $repo->{url} to $repo->{path}");
|
|
if ($repo->{pull_hook}) {
|
|
info("Running pull hook for $repo->{fullname}");
|
|
`$repo->{pull_hook}`;
|
|
}
|
|
($? != 0) and error("Failed to execute pull hook for $repo->{fullname}");
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
sub read_conf() {
|
|
my $configdir;
|
|
if ($ENV{XDG_CONFIG_HOME}) {
|
|
$configdir = "$ENV{XDG_CONFIG_HOME}/clonedev";
|
|
} else {
|
|
$configdir = "$ENV{HOME}/.config/clonedev";
|
|
}
|
|
open(my $cfg, '<', $configdir . "/config.yml") or die;
|
|
my $hashref = YAML::Load(do {local $/; <$cfg>});
|
|
close($cfg);
|
|
if (!$hashref->{hook_dir}) {
|
|
$hashref->{hook_dir} = "$configdir/hooks";
|
|
}
|
|
%conf = %$hashref;
|
|
}
|
|
|
|
|
|
sub full_pipeline($handle, $url, @headers) {
|
|
set_curl($handle, $url, @headers);
|
|
exec_curl($handle);
|
|
json_decode($handle);
|
|
url_filter($handle);
|
|
process_urls($handle);
|
|
handle_repos($handle);
|
|
}
|
|
|
|
# ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░
|
|
# ░ ░░░░ ░░░ ░░░ ░░ ░░░ ░
|
|
# ▒ ▒▒ ▒▒ ▒▒▒▒ ▒▒▒▒▒ ▒▒▒▒▒ ▒▒ ▒
|
|
# ▓ ▓▓ ▓▓▓▓ ▓▓▓▓▓ ▓▓▓▓▓ ▓ ▓ ▓
|
|
# █ █ █ ██ █████ █████ ██ █
|
|
# █ ████ ██ ████ ██ ██ ███ █
|
|
# ████████████████████████████████████████
|
|
|
|
read_conf();
|
|
|
|
my $last_handle = 0;
|
|
|
|
for my $i (keys @{$conf{lookups}}) {
|
|
my %lookup = %{$conf{lookups}[$i]};
|
|
chomp($ENV{TOKEN} = $lookup{token_cmd} ? `$lookup{token_cmd}` : "");
|
|
for (@{$lookup{extra_headers}}) {
|
|
$_ = `printf "$_"`;
|
|
}
|
|
if ($lookup{targets}) {
|
|
for my $j (keys @{$lookup{targets}}) {
|
|
$last_handle++;
|
|
$handles[$last_handle]{lookup} = $i;
|
|
full_pipeline($last_handle, "$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}", @{$lookup{extra_headers}});
|
|
}
|
|
} else {
|
|
$last_handle++;
|
|
$handles[$last_handle]{lookup} = $i;
|
|
full_pipeline($last_handle, "$lookup{api_url}/$lookup{endpoint}", @{$lookup{extra_headers}});
|
|
}
|
|
}
|
|
|
|
$last_handle++;
|
|
inject_conf_urls($last_handle);
|
|
process_urls($last_handle);
|
|
handle_repos($last_handle);
|
|
|
|
for my $message (@messages) {
|
|
print($message);
|
|
}
|