#!/bin/perl use strict; use warnings; use feature ("signatures"); use JSON; use WWW::Curl::Easy; use File::Path("make_path"); use Data::Dumper; $Data::Dumper::Pair = " : "; $Data::Dumper::Indent = 2; use YAML; $YAML::Preserve = 1; # TODO: Show hook/clone output in a prettier fashion (like docker buildx) # TODO: Allow branch selection # TODO: Allow updating a single repo # TODO: Add flags to allow checking for unclean trees, only cloning, only pulling # TODO: Check if directories are empty before cloning any repos to allow cloning in any order use constant USERAGENT => "User-Agent: MarxBot/4.2.0 (A script reading some information about repos)"; use constant URL_REGEX => qr/^((.*(?:@|:\/\/))[a-zA-Z0-9-_.]+(?:\/|:)(?:[a-zA-Z0-9-_.\/]+\/)?([a-zA-Z0-9-_.]+?)\/([a-zA-Z0-9-_.]+?)(?:\.git)?\/?)$/; my @handles; my @messages; my %conf; my $active_repos = 0; my $active_requests = 0; sub info($message) { print(`tput setaf 4; tput bold; echo '$message'; tput sgr 0`); } sub error($message) { push(@messages, `tput setaf 1; tput bold; echo '$message'; tput sgr 0`); } sub warning($message) { push(@messages, `tput setaf 3; tput bold; echo '$message'; tput sgr 0`); } sub set_curl($handle, $url, @headers) { $handles[$handle]{curl} = WWW::Curl::Easy->new; $handles[$handle]{curl}->setopt(CURLOPT_URL, $url); $handles[$handle]{curl}->pushopt(CURLOPT_HTTPHEADER, [USERAGENT]); for my $header (@headers) { $handles[$handle]{curl}->pushopt(CURLOPT_HTTPHEADER, [$header]); } $handles[$handle]{curl}->setopt(CURLOPT_PRIVATE, $handle); $handles[$handle]{curl}->setopt(CURLOPT_WRITEDATA, \$handles[$handle]{memory}); } sub add_callback($handle, $callback) { push(@{$handles[$handle]{callbacks}}, $callback); } sub exec_curl($handle) { my $curl = $handles[$handle]{curl}; $curl->perform; my $status = $curl->getinfo(CURLINFO_HTTP_CODE); if ($status < 200 || $status > 300) { my $url = $curl->getinfo(CURLOPT_URL); error("Curl on $url failed with code $status"); } } sub json_decode($handle) { $handles[$handle]{memory} = JSON::decode_json($handles[$handle]{memory}); } sub url_filter($handle) { my @tmp; my $lookup = $conf{lookups}[$handles[$handle]->{lookup}]; for my $repo (@{$handles[$handle]{memory}}) { if ($repo->{$lookup->{url_field}}) { push(@tmp, $repo->{$lookup->{url_field}}); } else { error("Failed to extract $lookup->{url_field} while processing lookup: $lookup->{name}"); } } $handles[$handle]{memory} = \@tmp; } sub inject_conf_urls($handle) { for my $url (@{$conf{extra_urls}}) { push(@{$handles[$handle]{memory}}, $url); } } sub process_urls($handle) { my @tmp; for my $url (@{$handles[$handle]{memory}}) { my %repo; if ($url =~ URL_REGEX) { $repo{url} = $1; $repo{owner} = $3; $repo{name} = $4; $repo{fullname} = "$3/$4"; if (substr($2, -1) eq "@") { $repo{protocol} = "ssh"; } else { $repo{protocol} = $2; } } else { error("Failed to parse url: $url"); next; } next if (grep($_ eq $repo{fullname}, @{$conf{skip_repos}})); my $path; my $repodir = $repo{name}; for my $pt (@{$conf{path_transformations}}) { if ($repo{fullname} =~ qr/$pt->{match}/) { $repodir =~ s/$pt->{replace}/$pt->{with}/; } } for my $directory (@{$conf{directories}}) { for my $regex (@{$directory->{repos}}) { if ( $repo{fullname} =~ qr/^$regex$/ && (!$directory->{lookups} || grep(qr/$conf{lookups}[ $handles[$handle]{lookup} ]{name}/, @{$directory->{lookups}})) ) { $repo{path} = `printf $directory->{path}/$repodir`; } } } if (!$repo{path} && !$conf{lookups}[$handles[$handle]{lookup}]{block_unsorted}) { $repo{path} = `printf $conf{unsorted_directory}/$repodir`; } elsif ($repo{path}) { } else { next; } my $clone_hook = "$conf{hook_dir}/clone/$repo{owner}:$repo{name}"; my $pull_hook = "$conf{hook_dir}/pull/$repo{owner}:$repo{name}"; (-x $clone_hook) and $repo{clone_hook} = "cd $repo{path} && $clone_hook"; (-x $pull_hook) and $repo{pull_hook} = "cd $repo{path} && $pull_hook"; push(@tmp, \%repo); } $handles[$handle]{memory} = \@tmp; } sub dump_mem($handle) { print("------ Handle $handle ------\n"); print Dumper($handles[$handle]->{memory}); } sub folder_is_empty($directory) { opendir(my $dh, $directory) or return 1; return scalar(grep($_ ne "." && $_ ne "..", readdir($dh))) == 0; } sub handle_repos($handle) { for my $repo (@{$handles[$handle]->{memory}}) { if (folder_is_empty("$repo->{path}")) { make_path($repo->{path}); info("Cloning $repo->{fullname}"); `git -C '$repo->{path}' clone $conf{clone_flags} '$repo->{url}' .`; ($? != 0) and error("Failed to clone $repo->{url} to $repo->{path}"); if ($repo->{clone_hook}) { info("Running clone hook for $repo->{fullname}"); `$repo->{clone_hook}`; } ($? != 0) and error("Failed to execute clone hook for $repo->{fullname}"); } elsif (!folder_is_empty("$repo->{path}/.git")) { info("Pulling $repo->{fullname} to $repo->{path}"); if (`git -C $repo->{path} status -z`) { warning("$repo->{path} has an unclean tree."); } else { `git -C $repo->{path} pull $conf{pull_flags}`; } ($? != 0) and error("Failed to pull $repo->{url} to $repo->{path}"); if ($repo->{pull_hook}) { info("Running pull hook for $repo->{fullname}"); `$repo->{pull_hook}`; } ($? != 0) and error("Failed to execute pull hook for $repo->{fullname}"); } } } sub read_conf() { my $configdir; if ($ENV{XDG_CONFIG_HOME}) { $configdir = "$ENV{XDG_CONFIG_HOME}/clonedev"; } else { $configdir = "$ENV{HOME}/.config/clonedev"; } open(my $cfg, '<', $configdir . "/config.yml") or die; my $hashref = YAML::Load(do {local $/; <$cfg>}); close($cfg); if (!$hashref->{hook_dir}) { $hashref->{hook_dir} = "$configdir/hooks"; } %conf = %$hashref; } sub full_pipeline($handle, $url, @headers) { set_curl($handle, $url, @headers); exec_curl($handle); json_decode($handle); url_filter($handle); process_urls($handle); handle_repos($handle); } # ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ # ░ ░░░░ ░░░ ░░░ ░░ ░░░ ░ # ▒ ▒▒ ▒▒ ▒▒▒▒ ▒▒▒▒▒ ▒▒▒▒▒ ▒▒ ▒ # ▓ ▓▓ ▓▓▓▓ ▓▓▓▓▓ ▓▓▓▓▓ ▓ ▓ ▓ # █ █ █ ██ █████ █████ ██ █ # █ ████ ██ ████ ██ ██ ███ █ # ████████████████████████████████████████ read_conf(); my $last_handle = 0; for my $i (keys @{$conf{lookups}}) { my %lookup = %{$conf{lookups}[$i]}; chomp($ENV{TOKEN} = $lookup{token_cmd} ? `$lookup{token_cmd}` : ""); for (@{$lookup{extra_headers}}) { $_ = `printf "$_"`; } if ($lookup{targets}) { for my $j (keys @{$lookup{targets}}) { $last_handle++; $handles[$last_handle]{lookup} = $i; full_pipeline($last_handle, "$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}", @{$lookup{extra_headers}}); } } else { $last_handle++; $handles[$last_handle]{lookup} = $i; full_pipeline($last_handle, "$lookup{api_url}/$lookup{endpoint}", @{$lookup{extra_headers}}); } } $last_handle++; inject_conf_urls($last_handle); process_urls($last_handle); handle_repos($last_handle); for my $message (@messages) { print($message); }