#!/bin/perl use strict; use warnings; use feature ("signatures"); use YAML(); use JSON(); use WWW::Curl::Easy; use WWW::Curl::Multi; use File::Path("make_path"); use Data::Dumper; $Data::Dumper::Pair = " : "; $Data::Dumper::Indent = 2; # TODO: Async the git clones # TODO: Show hook/clone output in a prettier fashion (like docker buildx) use constant USERAGENT => "User-Agent: MarxBot/4.2.0 (A script reading some information about repos)"; my @handles; my @messages; my %conf; my $active_repos = 0; my $active_requests = 0; sub info($message) { print(`tput setaf 4; tput bold; echo '$message'; tput sgr 0`); } sub error($message) { push( @messages, `tput setaf 1; tput bold; echo '$message'; tput sgr 0` ); } sub warning($message) { push( @messages, `tput setaf 3; tput bold; echo '$message'; tput sgr 0` ); } sub set_curl( $handle, $url, @headers ) { $handles[$handle]{curl} = WWW::Curl::Easy->new; $handles[$handle]{curl}->setopt( CURLOPT_URL, $url ); $handles[$handle]{curl}->pushopt( CURLOPT_HTTPHEADER, [USERAGENT] ); for my $header (@headers) { $handles[$handle]{curl}->pushopt( CURLOPT_HTTPHEADER, [$header] ); } $handles[$handle]{curl}->setopt( CURLOPT_PRIVATE, $handle ); $handles[$handle]{curl} ->setopt( CURLOPT_WRITEDATA, \$handles[$handle]{memory} ); } sub add_callback( $handle, $callback ) { push( @{ $handles[$handle]{callbacks} }, $callback ); } sub exec_callbacks($handle) { if ( $handles[$handle]{callbacks} ) { for my $callback ( @{ $handles[$handle]->{callbacks} } ) { $callback->($handle); } } } sub exec_multicurl() { my $curlm = WWW::Curl::Multi->new; for my $handle (@handles) { if ($handle) { $curlm->add_handle( $handle->{curl} ); $active_requests++; } } while ($active_requests) { my $active_transfers = $curlm->perform; if ( $active_transfers != $active_requests ) { while ( my ( $handle, $ret ) = $curlm->info_read ) { if ($handle) { $active_requests--; exec_callbacks($handle); # TODO: proper error checking # $handles[$handle]{curl}->getinfo(CURLINFO_HTTP_CODE); delete $handles[$handle]; } } } } } sub json_decode($handle) { $handles[$handle]{memory} = JSON::decode_json( $handles[$handle]{memory} ); } sub url_filter($handle) { my @tmp; my $lookup = $conf{lookups}[ $handles[$handle]->{lookup} ]; for my $repo ( @{ $handles[$handle]{memory} } ) { if ( $repo->{ $lookup->{url_field} } ) { push( @tmp, $repo->{ $lookup->{url_field} } ); } else { error( "Failed to extract $lookup->{url_field} while processing lookup: $lookup->{name}" ); } } $handles[$handle]{memory} = \@tmp; } sub inject_conf_urls($handle) { for my $url ( @{ $conf{extra_urls} } ) { push( @{ $handles[$handle]{memory} }, $url ); } } sub process_urls($handle) { my @tmp; for my $url ( @{ $handles[$handle]{memory} } ) { my %repo; if ( $url =~ /^((.*(?:@|:\/\/))[a-zA-Z0-9-_.]+(?:\/|:)(?:[a-zA-Z0-9-_.\/]+\/)?([a-zA-Z0-9-_.]+?)\/([a-zA-Z0-9-_.]+?)(?:\.git)?\/?)$/ ) { $repo{url} = $1; $repo{owner} = $3; $repo{name} = $4; $repo{fullname} = "$3/$4"; if ( substr( $2, -1 ) eq "@" ) { $repo{protocol} = "ssh"; } else { $repo{protocol} = $2; } } else { error("Failed to parse url: $url"); next; } next if ( grep( $_ eq $repo{fullname}, @{ $conf{skip_repos} } ) ); my $path; DIRS: for my $directory ( @{ $conf{directories} } ) { for my $regex ( @{ $directory->{repos} } ) { if ( $repo{fullname} =~ /$regex/ ) { $repo{path} = `printf $directory->{path}/$repo{name}`; last DIRS; } } } if (!$repo{path} && $conf{lookups}[ $handles[$handle]{lookup} ]{block_unsorted}) { $repo{path} = `printf $conf{unsorted_directory}/$repo{name}`; } my $clone_hook = "$conf{hook_dir}/clone/$repo{owner}:$repo{name}"; my $pull_hook = "$conf{hook_dir}/pull/$repo{owner}:$repo{name}"; ( -x $clone_hook ) and $repo{clone_hook} = "cd $repo{path} && $clone_hook"; ( -x $pull_hook ) and $repo{pull_hook} = "cd $repo{path} && $pull_hook"; push( @tmp, \%repo ); } $handles[$handle]{memory} = \@tmp; } sub dump($handle) { print("------ Handle $handle ------\n"); print Dumper( $handles[$handle]->{memory} ); } sub folder_is_empty($directory) { opendir( my $dh, $directory ) or return 1; return scalar( grep( $_ ne "." && $_ ne "..", readdir($dh) ) ) == 0; } sub handle_repos($handle) { for my $repo ( @{ $handles[$handle]->{memory} } ) { if ( folder_is_empty("$repo->{path}") ) { make_path( $repo->{path} ); info("Cloning $repo->{fullname}"); `git -C '$repo->{path}' clone $conf{clone_flags} '$repo->{url}' .`; ( $? != 0 ) and error("Failed to clone $repo->{url} to $repo->{path}"); if ( $repo->{clone_hook} ) { info("Running clone hook for $repo->{fullname}"); `$repo->{clone_hook}`; } ( $? != 0 ) and error("Failed to execute clone hook for $repo->{fullname}"); } elsif ( !folder_is_empty("$repo->{path}/.git") ) { info("Pulling $repo->{fullname} to $repo->{path}"); if (`git -C $repo->{path} status -z`) { warn("$repo->{path} has an unclean tree."); } else { `git -C $repo->{path} pull $conf{pull_flags}`; } ( $? != 0 ) and error("Failed to pull $repo->{url} to $repo->{path}"); if ( $repo->{pull_hook} ) { info("Running pull hook for $repo->{fullname}"); `$repo->{pull_hook}`; } ( $? != 0 ) and error("Failed to execute pull hook for $repo->{fullname}"); } } } sub read_conf() { my $configdir; if ( $ENV{XDG_CONFIG_HOME} ) { $configdir = "$ENV{XDG_CONFIG_HOME}/clonedev"; } else { $configdir = "$ENV{HOME}/.config/clonedev"; } open( my $cfg, '<', $configdir . "/config.yml" ) or die; my $hashref = YAML::Load( do { local $/; <$cfg> } ); close($cfg); if ( !$hashref->{hook_dir} ) { $hashref->{hook_dir} = "$configdir/hooks"; } for my $dir (@{$hashref->{directories}}) { grep(s/\//\\\//, @{$dir->{repos}}); } %conf = %$hashref; } sub curl_pipeline($handle) { add_callback( $handle, \&json_decode ); add_callback( $handle, \&url_filter ); add_callback( $handle, \&process_urls ); add_callback($handle, \&dump); # add_callback( $handle, \&handle_repos ); } # ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ # ░ ░░░░ ░░░ ░░░ ░░ ░░░ ░ # ▒ ▒▒ ▒▒ ▒▒▒▒ ▒▒▒▒▒ ▒▒▒▒▒ ▒▒ ▒ # ▓ ▓▓ ▓▓▓▓ ▓▓▓▓▓ ▓▓▓▓▓ ▓ ▓ ▓ # █ █ █ ██ █████ █████ ██ █ # █ ████ ██ ████ ██ ██ ███ █ # ████████████████████████████████████████ read_conf(); my $last_handle = 0; for my $i ( keys @{ $conf{lookups} } ) { my %lookup = %{ $conf{lookups}[$i] }; chomp( $ENV{TOKEN} = $lookup{token_cmd} ? `$lookup{token_cmd}` : "" ); for ( $lookup{extra_headers} ) { $_ = `printf "$_"`; } if ($lookup{targets}) { for my $j ( keys @{ $lookup{targets} } ) { $last_handle++; $handles[$last_handle]{lookup} = $i; set_curl( $last_handle, "$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}", $lookup{extra_headers} ); curl_pipeline($last_handle); } } else { $last_handle++; $handles[$last_handle]{lookup} = $i; set_curl( $last_handle, "$lookup{api_url}/$lookup{endpoint}", $lookup{extra_headers} ); curl_pipeline($last_handle); } } exec_multicurl(); $last_handle++; add_callback( $last_handle, \&inject_conf_urls ); add_callback( $last_handle, \&process_urls ); add_callback( $last_handle, \&handle_repos ); exec_callbacks($last_handle); for my $message (@messages) { print($message); }