1
0
Fork 0

clonedev reformatting, header fixes

This commit is contained in:
Luca Bilke 2024-02-27 13:39:46 +01:00
parent 88673923e6
commit fd0aeda19e
1 changed files with 32 additions and 43 deletions

View File

@ -16,13 +16,12 @@ $Data::Dumper::Indent = 2;
# TODO: Async the git clones # TODO: Async the git clones
# TODO: Show hook/clone output in a prettier fashion (like docker buildx) # TODO: Show hook/clone output in a prettier fashion (like docker buildx)
use constant USERAGENT => use constant USERAGENT => "User-Agent: MarxBot/4.2.0 (A script reading some information about repos)";
"User-Agent: MarxBot/4.2.0 (A script reading some information about repos)";
my @handles; my @handles;
my @messages; my @messages;
my %conf; my %conf;
my $active_repos = 0; my $active_repos = 0;
my $active_requests = 0; my $active_requests = 0;
sub info($message) { sub info($message) {
@ -45,8 +44,7 @@ sub set_curl( $handle, $url, @headers ) {
$handles[$handle]{curl}->pushopt( CURLOPT_HTTPHEADER, [$header] ); $handles[$handle]{curl}->pushopt( CURLOPT_HTTPHEADER, [$header] );
} }
$handles[$handle]{curl}->setopt( CURLOPT_PRIVATE, $handle ); $handles[$handle]{curl}->setopt( CURLOPT_PRIVATE, $handle );
$handles[$handle]{curl} $handles[$handle]{curl}->setopt( CURLOPT_WRITEDATA, \$handles[$handle]{memory} );
->setopt( CURLOPT_WRITEDATA, \$handles[$handle]{memory} );
} }
sub add_callback( $handle, $callback ) { sub add_callback( $handle, $callback ) {
@ -95,11 +93,8 @@ sub url_filter($handle) {
for my $repo ( @{ $handles[$handle]{memory} } ) { for my $repo ( @{ $handles[$handle]{memory} } ) {
if ( $repo->{ $lookup->{url_field} } ) { if ( $repo->{ $lookup->{url_field} } ) {
push( @tmp, $repo->{ $lookup->{url_field} } ); push( @tmp, $repo->{ $lookup->{url_field} } );
} } else {
else { error( "Failed to extract $lookup->{url_field} while processing lookup: $lookup->{name}" );
error(
"Failed to extract $lookup->{url_field} while processing lookup: $lookup->{name}"
);
} }
} }
$handles[$handle]{memory} = \@tmp; $handles[$handle]{memory} = \@tmp;
@ -115,10 +110,7 @@ sub process_urls($handle) {
my @tmp; my @tmp;
for my $url ( @{ $handles[$handle]{memory} } ) { for my $url ( @{ $handles[$handle]{memory} } ) {
my %repo; my %repo;
if ( $url =~ if ( $url =~ /^((.*(?:@|:\/\/))[a-zA-Z0-9-_.]+(?:\/|:)(?:[a-zA-Z0-9-_.\/]+\/)?([a-zA-Z0-9-_.]+?)\/([a-zA-Z0-9-_.]+?)(?:\.git)?\/?)$/ ) {
/^((.*(?:@|:\/\/))[a-zA-Z0-9-_.]+(?:\/|:)(?:[a-zA-Z0-9-_.\/]+\/)?([a-zA-Z0-9-_.]+?)\/([a-zA-Z0-9-_.]+?)(?:\.git)?\/?)$/
)
{
$repo{url} = $1; $repo{url} = $1;
$repo{owner} = $3; $repo{owner} = $3;
$repo{name} = $4; $repo{name} = $4;
@ -126,12 +118,10 @@ sub process_urls($handle) {
if ( substr( $2, -1 ) eq "@" ) { if ( substr( $2, -1 ) eq "@" ) {
$repo{protocol} = "ssh"; $repo{protocol} = "ssh";
} } else {
else {
$repo{protocol} = $2; $repo{protocol} = $2;
} }
} } else {
else {
error("Failed to parse url: $url"); error("Failed to parse url: $url");
next; next;
} }
@ -148,14 +138,15 @@ sub process_urls($handle) {
} }
} }
if (!$repo{path} && $conf{lookups}[ $handles[$handle]{lookup} ]{block_unsorted}) { if ( !$repo{path} && !$conf{lookups}[ $handles[$handle]{lookup} ]{block_unsorted} ) {
$repo{path} = `printf $conf{unsorted_directory}/$repo{name}`; $repo{path} = `printf $conf{unsorted_directory}/$repo{name}`;
} else {
next;
} }
my $clone_hook = "$conf{hook_dir}/clone/$repo{owner}:$repo{name}"; my $clone_hook = "$conf{hook_dir}/clone/$repo{owner}:$repo{name}";
my $pull_hook = "$conf{hook_dir}/pull/$repo{owner}:$repo{name}"; my $pull_hook = "$conf{hook_dir}/pull/$repo{owner}:$repo{name}";
( -x $clone_hook ) ( -x $clone_hook ) and $repo{clone_hook} = "cd $repo{path} && $clone_hook";
and $repo{clone_hook} = "cd $repo{path} && $clone_hook";
( -x $pull_hook ) and $repo{pull_hook} = "cd $repo{path} && $pull_hook"; ( -x $pull_hook ) and $repo{pull_hook} = "cd $repo{path} && $pull_hook";
push( @tmp, \%repo ); push( @tmp, \%repo );
} }
@ -178,31 +169,26 @@ sub handle_repos($handle) {
make_path( $repo->{path} ); make_path( $repo->{path} );
info("Cloning $repo->{fullname}"); info("Cloning $repo->{fullname}");
`git -C '$repo->{path}' clone $conf{clone_flags} '$repo->{url}' .`; `git -C '$repo->{path}' clone $conf{clone_flags} '$repo->{url}' .`;
( $? != 0 ) ( $? != 0 ) and error("Failed to clone $repo->{url} to $repo->{path}");
and error("Failed to clone $repo->{url} to $repo->{path}");
if ( $repo->{clone_hook} ) { if ( $repo->{clone_hook} ) {
info("Running clone hook for $repo->{fullname}"); info("Running clone hook for $repo->{fullname}");
`$repo->{clone_hook}`; `$repo->{clone_hook}`;
} }
( $? != 0 ) ( $? != 0 ) and error("Failed to execute clone hook for $repo->{fullname}");
and error("Failed to execute clone hook for $repo->{fullname}");
} }
elsif ( !folder_is_empty("$repo->{path}/.git") ) { elsif ( !folder_is_empty("$repo->{path}/.git") ) {
info("Pulling $repo->{fullname} to $repo->{path}"); info("Pulling $repo->{fullname} to $repo->{path}");
if (`git -C $repo->{path} status -z`) { if (`git -C $repo->{path} status -z`) {
warn("$repo->{path} has an unclean tree."); warn("$repo->{path} has an unclean tree.");
} } else {
else {
`git -C $repo->{path} pull $conf{pull_flags}`; `git -C $repo->{path} pull $conf{pull_flags}`;
} }
( $? != 0 ) ( $? != 0 ) and error("Failed to pull $repo->{url} to $repo->{path}");
and error("Failed to pull $repo->{url} to $repo->{path}");
if ( $repo->{pull_hook} ) { if ( $repo->{pull_hook} ) {
info("Running pull hook for $repo->{fullname}"); info("Running pull hook for $repo->{fullname}");
`$repo->{pull_hook}`; `$repo->{pull_hook}`;
} }
( $? != 0 ) ( $? != 0 ) and error("Failed to execute pull hook for $repo->{fullname}");
and error("Failed to execute pull hook for $repo->{fullname}");
} }
} }
} }
@ -211,8 +197,7 @@ sub read_conf() {
my $configdir; my $configdir;
if ( $ENV{XDG_CONFIG_HOME} ) { if ( $ENV{XDG_CONFIG_HOME} ) {
$configdir = "$ENV{XDG_CONFIG_HOME}/clonedev"; $configdir = "$ENV{XDG_CONFIG_HOME}/clonedev";
} } else {
else {
$configdir = "$ENV{HOME}/.config/clonedev"; $configdir = "$ENV{HOME}/.config/clonedev";
} }
open( my $cfg, '<', $configdir . "/config.yml" ) or die; open( my $cfg, '<', $configdir . "/config.yml" ) or die;
@ -221,8 +206,8 @@ sub read_conf() {
if ( !$hashref->{hook_dir} ) { if ( !$hashref->{hook_dir} ) {
$hashref->{hook_dir} = "$configdir/hooks"; $hashref->{hook_dir} = "$configdir/hooks";
} }
for my $dir (@{$hashref->{directories}}) { for my $dir ( @{ $hashref->{directories} } ) {
grep(s/\//\\\//, @{$dir->{repos}}); grep( s/\//\\\//, @{ $dir->{repos} } );
} }
%conf = %$hashref; %conf = %$hashref;
} }
@ -231,7 +216,6 @@ sub curl_pipeline($handle) {
add_callback( $handle, \&json_decode ); add_callback( $handle, \&json_decode );
add_callback( $handle, \&url_filter ); add_callback( $handle, \&url_filter );
add_callback( $handle, \&process_urls ); add_callback( $handle, \&process_urls );
# add_callback($handle, \&dump);
add_callback( $handle, \&handle_repos ); add_callback( $handle, \&handle_repos );
} }
@ -250,24 +234,29 @@ my $last_handle = 0;
for my $i ( keys @{ $conf{lookups} } ) { for my $i ( keys @{ $conf{lookups} } ) {
my %lookup = %{ $conf{lookups}[$i] }; my %lookup = %{ $conf{lookups}[$i] };
chomp( $ENV{TOKEN} = $lookup{token_cmd} ? `$lookup{token_cmd}` : "" ); chomp( $ENV{TOKEN} = $lookup{token_cmd} ? `$lookup{token_cmd}` : "" );
for ( $lookup{extra_headers} ) { for ( @{ $lookup{extra_headers} } ) {
$_ = `printf "$_"`; $_ = `printf "$_"`;
} }
if ($lookup{targets}) { if ( $lookup{targets} ) {
for my $j ( keys @{ $lookup{targets} } ) { for my $j ( keys @{ $lookup{targets} } ) {
$last_handle++; $last_handle++;
$handles[$last_handle]{lookup} = $i; $handles[$last_handle]{lookup} = $i;
set_curl( $last_handle, set_curl(
$last_handle,
"$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}", "$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}",
$lookup{extra_headers} ); @{ $lookup{extra_headers} }
);
curl_pipeline($last_handle); curl_pipeline($last_handle);
} }
} else { }
else {
$last_handle++; $last_handle++;
$handles[$last_handle]{lookup} = $i; $handles[$last_handle]{lookup} = $i;
set_curl( $last_handle, set_curl(
$last_handle,
"$lookup{api_url}/$lookup{endpoint}", "$lookup{api_url}/$lookup{endpoint}",
$lookup{extra_headers} ); @{ $lookup{extra_headers} }
);
curl_pipeline($last_handle); curl_pipeline($last_handle);
} }
} }