Original release 20160919

[fex.git] / bin / fex_cleanup

diff --git a/bin/fex_cleanup b/bin/fex_cleanup
index c54b2e0d832852ecbb121fa5b17290a9c3361498..1f38d6b42b5bc975b692e76d026a09afacf8779b 100755 (executable)
--- a/bin/fex_cleanup
+++ b/bin/fex_cleanup
@@ -1,4 +1,4 @@
-#!/usr/bin/perl -w 
+#!/usr/bin/perl -w

 
 # cleanup for F*EX service
 #
 
 # cleanup for F*EX service
 #


@@ -14,7 +14,7 @@ use Cwd       'abs_path';
 use Digest::MD5        'md5_hex';
 
 use constant DS => 60*60*24;
 use Digest::MD5        'md5_hex';
 
 use constant DS => 60*60*24;

-  
+

 # do not run as CGI!
 exit if $ENV{SCRIPT_NAME};
 
 # do not run as CGI!
 exit if $ENV{SCRIPT_NAME};
 


@@ -38,15 +38,18 @@ $| = 1;
 
 # use fex.ph for site configuration!
 our ($FEXHOME);
 
 # use fex.ph for site configuration!
 our ($FEXHOME);

-our ($spooldir,$logdir,$docdir);
+our ($spooldir,@logdir,$docdir);

 our ($akeydir,$ukeydir,$dkeydir,$skeydir,$gkeydir,$xkeydir,$lockdir);
 our ($durl,$debug,$autodelete,$hostname,$admin,$admin_pw,$bcc);
 our ($akeydir,$ukeydir,$dkeydir,$skeydir,$gkeydir,$xkeydir,$lockdir);
 our ($durl,$debug,$autodelete,$hostname,$admin,$admin_pw,$bcc);

-$keep_default = 5;
+our $keep_default = 5;
+our $purge = $keep_default*3;

 
 # load common code, local config : $HOME/lib/fex.ph
 require "$FEXLIB/fex.pp" or die "$0: cannot load $FEXLIB/fex.pp - $!\n";
 
 
 # load common code, local config : $HOME/lib/fex.ph
 require "$FEXLIB/fex.pp" or die "$0: cannot load $FEXLIB/fex.pp - $!\n";
 

-# localized functions 
+my $logdir = $logdir[0];
+
+# localized functions

 # (needed for reminder and account reactivation e-mails)
 foreach my $lf (glob "$FEXHOME/locale/*/lib/lf.pl") { require $lf }
 
 # (needed for reminder and account reactivation e-mails)
 foreach my $lf (glob "$FEXHOME/locale/*/lib/lf.pl") { require $lf }
 


@@ -64,65 +67,65 @@ $today = time;
 $isodate = isodate($today);
 
 chdir $spooldir or die "$0: $spooldir - $!\n";
 $isodate = isodate($today);
 
 chdir $spooldir or die "$0: $spooldir - $!\n";

-open L,">>$logdir/cleanup.log";
+# open L,">>$logdir/cleanup.log";

 
 # clean up regular spool
 opendir $spooldir,'.' or die "$0: $spooldir - $!\n";
 while ($to = readdir $spooldir) {
 
 # clean up regular spool
 opendir $spooldir,'.' or die "$0: $spooldir - $!\n";
 while ($to = readdir $spooldir) {

-  next if $to !~ /@/ or -l $to;
-  if (@demo and -f "$to/.demo" and time > mtime("$to/.demo")+$demo[1]*DS) {
+  next if $to =~ /^\./;
+  next if $to !~ /@/ or $_ = readlink($to) and not /\//;
+  next unless -d $to;
+  if (@demo and -f "$to/.demo" and time > lmtime("$to/.demo")+$demo[1]*DS) {

     logdel($to,"demo user $to deleted");
     next;
   }
     logdel($to,"demo user $to deleted");
     next;
   }

-  if (-d $to and $to !~ /^\./) {
-    unless (opendir TO,$to) {
-      warn "$0: $spooldir/$to - $!\n";
-      next;
-    }
-    while ($from = readdir TO) {
-      next if $from !~ /@/;
-      if ($from eq '@GROUP') {
-        foreach $group (glob "$to/$from/*") {
-          if (readlink $group and not -f $group) {
-            logdel($group,"$group deleted (master has gone)");
-          }
+  unless (opendir TO,$to) {
+    warn "$0: $spooldir/$to - $!\n";
+    next;
+  }
+  while ($from = readdir TO) {
+    next if $from !~ /@/;
+    if ($from eq '@GROUP') {
+      foreach $group (glob "$to/$from/*") {
+        if (readlink $group and not -f $group) {
+          logdel($group,"$group deleted (master has gone)");

         }
         }

-      } else {
-        if (-d "$to/$from" and $from !~ /^\./) {
-          unless (opendir FROM,"$to/$from") {
-            warn "$0: $spooldir/$to/$from - $!\n";
-            next;
-          }
-          while ($file = readdir FROM) {
-            next if $file eq '.' or $file eq '..';
-            if (-d "$to/$from/$file" and $file !~ /^\./) {
-              cleanup($to,$from,$file);
-              rmdir "$to/$from/$file" unless $opt_d;
-            }
+      }
+    } else {
+      if (-d "$to/$from" and $from !~ /^\./) {
+        unless (opendir FROM,"$to/$from") {
+          warn "$0: $spooldir/$to/$from - $!\n";
+          next;
+        }
+        while ($file = readdir FROM) {
+          next if $file eq '.' or $file eq '..';
+          if (-d "$to/$from/$file" and $file !~ /^\./) {
+            cleanup($to,$from,$file);
+            rmdir "$to/$from/$file" unless $opt_d;

           }
           }

-          closedir FROM;
-          rmdir "$to/$from" unless $opt_d;

         }
         }

+        closedir FROM;
+        rmdir "$to/$from" unless $opt_d;

       }
     }
       }
     }

-    closedir TO;
-    unless (-f "$to/\@PERSISTENT" or $to eq $admin) {
-      @glob = glob "$to/*/* $to/\@MAINUSER/* $to/\@GROUP/*";
-      unless (@glob or -f "$to/\@") {
-        logdel($to,"$to deleted");
-      }
-      $user = $to;
-      if ($login_check and -l "$user/.login") {
-        my $lc = &$login_check(readlink("$user/.login"));
-        if ($lc) {
-          if (-f "$user/\@~" and not "$user/@") {
-            rename "$user/\@~","$user/@" unless $opt_d;
-            logv("$isodate $user reanimated (login_check)");
-          }
-        } else {
-          rename "$user/@","$user/\@~" unless $opt_d;
-          logv("$user deactivated (login_check)");
+  }
+  closedir TO;
+  unless (-f "$to/\@PERSISTENT" or $to eq $admin) {
+    @glob = glob "$to/*/* $to/\@MAINUSER/* $to/\@GROUP/*";
+    unless (@glob or -f "$to/\@") {
+      logdel($to,"$to deleted");
+    }
+    $user = $to;
+    if ($login_check and -l "$user/.login") {
+      my $lc = &$login_check(readlink("$user/.login"));
+      if ($lc) {
+        if (-f "$user/\@~" and not "$user/@") {
+          rename "$user/\@~","$user/@" unless $opt_d;
+          logv("$user reanimated (login_check)");

         }
         }

+      } else {
+        rename "$user/@","$user/\@~" unless $opt_d;
+        logv("$user deactivated (login_check)");

       }
     }
   }
       }
     }
   }


@@ -132,7 +135,7 @@ closedir $spooldir;
 # clean up download key lookup directory
 if (chdir $dkeydir and opendir D,'.') {
   while ($file = readdir D) {
 # clean up download key lookup directory
 if (chdir $dkeydir and opendir D,'.') {
   while ($file = readdir D) {

-    if ($link = readlink $file and 
+    if ($link = readlink $file and

         (not -l "$link/dkey" or readlink "$link/dkey" ne $file)) {
       logdel($file,".dkeys/$file deleted");
     }
         (not -l "$link/dkey" or readlink "$link/dkey" ne $file)) {
       logdel($file,".dkeys/$file deleted");
     }


@@ -145,7 +148,7 @@ if (chdir $ukeydir and opendir D,'.') {
   while ($file = readdir D) {
     next if $file eq '.' or $file eq '..';
     if (($link = readlink $file and not -e "$link/upload"
   while ($file = readdir D) {
     next if $file eq '.' or $file eq '..';
     if (($link = readlink $file and not -e "$link/upload"

-         or -f $file and time > mtime($file)+DS)) {
+         or -f $file and time > lmtime($file)+DS)) {

       logdel($file,".ukeys/$file deleted");
     }
   }
       logdel($file,".ukeys/$file deleted");
     }
   }


@@ -155,7 +158,7 @@ if (chdir $ukeydir and opendir D,'.') {
 # clean up authorization key lookup directory
 if (chdir $akeydir and opendir D,'.') {
   while ($file = readdir D) {
 # clean up authorization key lookup directory
 if (chdir $akeydir and opendir D,'.') {
   while ($file = readdir D) {

-    if (-l $file and time > mtime($file)+DS) {
+    if (-l $file and time > (lmtime($file)||0)+DS) {

       logdel($file,".akeys/$file deleted");
     }
   }
       logdel($file,".akeys/$file deleted");
     }
   }


@@ -176,7 +179,7 @@ if (chdir $xkeydir and opendir D,'.') {
 # clean up lock directory
 if (chdir $lockdir and opendir D,'.') {
   while ($file = readdir D) {
 # clean up lock directory
 if (chdir $lockdir and opendir D,'.') {
   while ($file = readdir D) {

-    if (-f $file and time > mtime($file)+DS) {
+    if (-f $file and time > lmtime($file)+DS) {

       logdel($file,".locks/$file deleted");
     }
   }
       logdel($file,".locks/$file deleted");
     }
   }


@@ -187,7 +190,7 @@ if (chdir $lockdir and opendir D,'.') {
 if (chdir "$spooldir/.error" and opendir D,'.') {
   while ($file = readdir D) {
     if (-f $file) {
 if (chdir "$spooldir/.error" and opendir D,'.') {
   while ($file = readdir D) {
     if (-f $file) {

-      $mtime = mtime($file);
+      $mtime = lmtime($file);

       if ($mtime and $today > 10*$keep_default*DS+$mtime) {
         if ($opt_d) { print "unlink .error/$file\n" }
         else        { logdel($file,".error/$file deleted") }
       if ($mtime and $today > 10*$keep_default*DS+$mtime) {
         if ($opt_d) { print "unlink .error/$file\n" }
         else        { logdel($file,".error/$file deleted") }


@@ -198,10 +201,10 @@ if (chdir "$spooldir/.error" and opendir D,'.') {
 }
 
 # clean up debug directory
 }
 
 # clean up debug directory

-if (chdir "$logdir/.debug" and opendir D,'.') {
+if (chdir "$spooldir/.debug" and opendir D,'.') {

   while ($file = readdir D) {
     if (-f $file) {
   while ($file = readdir D) {
     if (-f $file) {

-      $mtime = mtime($file);
+      $mtime = lmtime($file);

       if ($mtime and $today > $keep_default*DS+$mtime) {
         # logdel($file,".debug/$file deleted");
         if ($opt_d) { print "unlink .debug/$file\n" }
       if ($mtime and $today > $keep_default*DS+$mtime) {
         # logdel($file,".debug/$file deleted");
         if ($opt_d) { print "unlink .debug/$file\n" }


@@ -257,7 +260,7 @@ foreach $subuser (glob '*/@MAINUSER') {
 # clean up old OKEYs
 chdir $spooldir;
 foreach my $okey (glob '*/@OKEY/*') {
 # clean up old OKEYs
 chdir $spooldir;
 foreach my $okey (glob '*/@OKEY/*') {

-  if (time > mtime($okey)+30*DS) {
+  if (time > lmtime($okey)+30*DS) {

     logdel($okey,"$okey deleted");
   }
 }
     logdel($okey,"$okey deleted");
   }
 }


@@ -301,7 +304,7 @@ if (chdir $gkeydir and opendir D,'.') {
 if (chdir "$spooldir/.reg" and opendir D,'.') {
   while ($file = readdir D) {
     if (-f $file) {
 if (chdir "$spooldir/.reg" and opendir D,'.') {
   while ($file = readdir D) {
     if (-f $file) {

-      $mtime = mtime($file);
+      $mtime = lmtime($file);

       if ($mtime and $today > $mtime+DS) {
         logdel($file,".reg/$file deleted");
       }
       if ($mtime and $today > $mtime+DS) {
         logdel($file,".reg/$file deleted");
       }


@@ -310,7 +313,7 @@ if (chdir "$spooldir/.reg" and opendir D,'.') {
   closedir D;
 }
 
   closedir D;
 }
 

-# send account expiration warning 
+# send account expiration warning

 if ($account_expire and $account_expire =~ /^(\d+)/) {
   my $expire = $1;
   if (chdir $spooldir) {
 if ($account_expire and $account_expire =~ /^(\d+)/) {
   my $expire = $1;
   if (chdir $spooldir) {


@@ -346,9 +349,9 @@ if ($account_expire and $account_expire =~ /^(\d+)/) {
       next if $user =~ /^(fexmaster|fexmail)/ or $user eq $admin;
       next if -l "$user/.login";
 
       next if $user =~ /^(fexmaster|fexmail)/ or $user eq $admin;
       next if -l "$user/.login";
 

-      if (time > mtime($user)+$expire*DS) {
+      if (time > lmtime($user)+$expire*DS) {

         # print "$spooldir/$user\n";
         # print "$spooldir/$user\n";

-        my $locale = readlink "$user/\@LOCALE";
+        local $locale = readlink "$user/\@LOCALE";

         $locale = 'english' unless $locale and $reactivation{$locale};
         &{$reactivation{$locale}}($expire,$user);
         sleep 1;
         $locale = 'english' unless $locale and $reactivation{$locale};
         &{$reactivation{$locale}}($expire,$user);
         sleep 1;


@@ -358,8 +361,6 @@ if ($account_expire and $account_expire =~ /^(\d+)/) {
   }
 }
 
   }
 }
 

-close L;
-

 # vhosts
 exit if $opt_V;
 if (%vhost) {
 # vhosts
 exit if $opt_V;
 if (%vhost) {


@@ -374,7 +375,8 @@ if (%vhost) {
   }
 }
 
   }
 }
 

-if ($notify_newrelease or not defined $notify_newrelease) {
+if ($notify_newrelease and $notify_newrelease !~ /^no$/i
+    or not defined $notify_newrelease) {

   $notify_newrelease ||= $admin;
   $newnew = $new = '';
   $snew = $FEXHOME.'/doc/new';
   $notify_newrelease ||= $admin;
   $newnew = $new = '';
   $snew = $FEXHOME.'/doc/new';


@@ -382,12 +384,13 @@ if ($notify_newrelease or not defined $notify_newrelease) {
   $_ = slurp("$FEXHOME/doc/version")||'';
   if (/(\d+)/) { $qn = "new?$hostname:$1" }
   else         { $qn = "new?$hostname:0" }
   $_ = slurp("$FEXHOME/doc/version")||'';
   if (/(\d+)/) { $qn = "new?$hostname:$1" }
   else         { $qn = "new?$hostname:0" }

+  print "checking for new F*EX release\n" if $opt_v;

   for (1..3) {
     sleep rand(10);
   for (1..3) {
     sleep rand(10);

-    $newnew = `wget -qO- http://fex.rus.uni-stuttgart.de/$qn 2>/dev/null`;
-    last if $newnew =~ /release/;

     $newnew = `wget -qO- http://fex.belwue.de/$qn 2>/dev/null`;
     last if $newnew =~ /release/;
     $newnew = `wget -qO- http://fex.belwue.de/$qn 2>/dev/null`;
     last if $newnew =~ /release/;

+    # $newnew = `wget -qO- http://fex.rus.uni-stuttgart.de/$qn 2>/dev/null`;
+    # last if $newnew =~ /release/;

   };
   if ($newnew =~ /release/) {
     if ($newnew ne $new) {
   };
   if ($newnew =~ /release/) {
     if ($newnew ne $new) {


@@ -416,12 +419,13 @@ exit;
 sub cleanup {
   my ($to,$from,$file) = @_;
   my ($data,$download,$notify,$mtime,$warn,$dir,$filename,$dkey,$delay);
 sub cleanup {
   my ($to,$from,$file) = @_;
   my ($data,$download,$notify,$mtime,$warn,$dir,$filename,$dkey,$delay);

-  my $comment = '';

   my $keep = $keep_default;
   my $keep = $keep_default;

+  my $purge = $::purge || 3*$keep;
+  my $comment = '';

   my $kf = "$to/$from/$file/keep";
   my $ef = "$to/$from/$file/error";
   local $_;
   my $kf = "$to/$from/$file/keep";
   my $ef = "$to/$from/$file/error";
   local $_;

-  
+

   $keep = readlink $kf || readlink "$to/\@KEEP" || $keep_default;
 
   $file       = "$to/$from/$file";
   $keep = readlink $kf || readlink "$to/\@KEEP" || $keep_default;
 
   $file       = "$to/$from/$file";


@@ -432,14 +436,15 @@ sub cleanup {
   if ($file =~ /\/ADDRESS_BOOK/) {
     logdel($file,"$file deleted");
   } elsif (-d $file and not -f $data) {
   if ($file =~ /\/ADDRESS_BOOK/) {
     logdel($file,"$file deleted");
   } elsif (-d $file and not -f $data) {

-    if ($mtime = mtime("$file/upload")) {
+    if ($mtime = lmtime("$file/upload")) {

       if ($today > $mtime+DS) {
         verbose("rmrf $file (today=$today mtime_upload=$mtime)");
         logdel($file,"$file deleted");
       }
       if ($today > $mtime+DS) {
         verbose("rmrf $file (today=$today mtime_upload=$mtime)");
         logdel($file,"$file deleted");
       }

-    } elsif ($mtime = mtime("$file/error")) {
-      if ($today > 3*$keep*DS+$mtime) {
-        verbose("rmrf $file (today=$today mtime_error=$mtime keep=$keep)");
+    } elsif ($mtime = lmtime("$file/error")) {
+      $purge = $1*$keep if $purge =~ /(\d+).*keep/;
+      if ($today > $purge*DS+$mtime) {
+        verbose("rmrf $file (today=$today mtime_error=$mtime keep=$keep purge=$purge)");

         logdel($file,"$file deleted");
       }
     } else {
         logdel($file,"$file deleted");
       }
     } else {


@@ -449,19 +454,19 @@ sub cleanup {
     $delay = autodelete($file);
     $delay = 1 if $delay !~ /^\d+$/;
     $delay--;
     $delay = autodelete($file);
     $delay = 1 if $delay !~ /^\d+$/;
     $delay--;

-    $mtime = mtime($download);
-    if ($mtime and $today > $delay*DS+$mtime 
+    $mtime = lmtime($download);
+    if ($mtime and $today > $delay*DS+$mtime

         and logdel($data,"$data deleted")) {
       if (open $ef,'>',$ef) {
         printf {$ef} "%s has been autodeleted after download at %s\n",
         and logdel($data,"$data deleted")) {
       if (open $ef,'>',$ef) {
         printf {$ef} "%s has been autodeleted after download at %s\n",

-                     filename($file),isodate(mtime($download));
+                     filename($file),isodate(lmtime($download));

         close $ef;
       }
     }
   } elsif (-f $data) {
     my $reactivation = $file =~ m{/\Q$admin/reactivation.txt\E$};
     $warn = $reactivation ? $keep-5 : $keep-2;
         close $ef;
       }
     }
   } elsif (-f $data) {
     my $reactivation = $file =~ m{/\Q$admin/reactivation.txt\E$};
     $warn = $reactivation ? $keep-5 : $keep-2;

-    $mtime = mtime("$file/filename") || mtime($data) || 0;
+    $mtime = lmtime("$file/filename") || lmtime($data) || 0;

     if ($today > $mtime+$keep*DS) {
       if ($account_expire and $reactivation) {
         if ($account_expire =~ /delete/) {
     if ($today > $mtime+$keep*DS) {
       if ($account_expire and $reactivation) {
         if ($account_expire =~ /delete/) {


@@ -490,7 +495,7 @@ sub cleanup {
           # also _fexmail_*
           logdel($file,"$file deleted") and
           verbose("rmrf $file (today=$today mtime_upload=$mtime)");
           # also _fexmail_*
           logdel($file,"$file deleted") and
           verbose("rmrf $file (today=$today mtime_upload=$mtime)");

-        } elsif (logdel($data,"$data deleted")) { 
+        } elsif (logdel($data,"$data deleted")) {

           verbose("unlink $data (today=$today mtime=$mtime keep=$keep)");
           if (open $ef,'>',$ef) {
             $filename = $file;
           verbose("unlink $data (today=$today mtime=$mtime keep=$keep)");
           if (open $ef,'>',$ef) {
             $filename = $file;


@@ -500,13 +505,13 @@ sub cleanup {
           }
         }
       }
           }
         }
       }

-    } 
+    }

     elsif ($file !~ /STDFEX$/ and
     elsif ($file !~ /STDFEX$/ and

-           $mtime+$warn*DS < $today and 
+           $mtime+$warn*DS < $today and

            $dkey = readlink("$file/dkey") and
            $dkey = readlink("$file/dkey") and

-           not -s $download and 
-           not -f $notify and 
-           (readlink("$to/\@REMINDER")||'yes') ne 'no') 
+           not -s $download and
+           not -f $notify and
+           (readlink("$to/\@REMINDER")||'yes') ne 'no')

     {
       my $locale = readlink "$to/\@LOCALE" || readlink "$file/\@LOCALE";
       $locale = 'english' unless $locale and $notify{$locale};
     {
       my $locale = readlink "$to/\@LOCALE" || readlink "$file/\@LOCALE";
       $locale = 'english' unless $locale and $notify{$locale};


@@ -514,7 +519,7 @@ sub cleanup {
         chomp ($comment = <$c>||'');
         close $c;
       }
         chomp ($comment = <$c>||'');
         close $c;
       }

-      &{$notify{$locale}}(
+      if (&{$notify{$locale}}(

         status     => 'remind',
         dkey       => $dkey,
         filename   => filename($file),
         status     => 'remind',
         dkey       => $dkey,
         filename   => filename($file),


@@ -522,10 +527,13 @@ sub cleanup {
         comment    => $comment,
         warn       => int(($mtime-$today)/DS)+$keep,
         autodelete => autodelete($file),
         comment    => $comment,
         warn       => int(($mtime-$today)/DS)+$keep,
         autodelete => autodelete($file),

-      );
-      open $notify,'>',$notify;
-      close $notify;
-      print "sent reminder for $file\n" if -t or $opt_v;
+      )) {
+        open $notify,'>',$notify;
+        close $notify;
+        print "sent reminder for $file\n" if -t or $opt_v;
+      } else {
+        warn "$0: reminder notification for $file failed\n";
+      }

     }
   }
 }
     }
   }
 }


@@ -541,13 +549,8 @@ sub autodelete {
     chomp($autodelete = <$adf>||'');
     close $adf;
   }
     chomp($autodelete = <$adf>||'');
     close $adf;
   }

-  
-  return $autodelete||$::autodelete;
-}

 
 

-sub mtime {
-  my @s = lstat shift;
-  return @s ? $s[9] : undef;
+  return $autodelete||$::autodelete;

 }
 
 sub logdel {
 }
 
 sub logdel {


@@ -560,19 +563,28 @@ sub logdel {
     if ($status = rmrf($file)) {
       logv($msg);
     } else {
     if ($status = rmrf($file)) {
       logv($msg);
     } else {

-      print L "$isodate $file DEL FAILED : $!\n";
-      warn     "$file DEL FAILED : $!\n" if -t or $opt_v;
+      logv("$file DEL FAILED : $!");
+      warn "$file DEL FAILED : $!\n" if -t or $opt_v;

     }
   }
     }
   }

-  
+

   return $status;
 }
 
 
 sub logv {
   my $msg = shift;
   return $status;
 }
 
 
 sub logv {
   my $msg = shift;

-  print L "$isodate $msg\n" unless $opt_d;
+

   print "$msg\n" if -t or $opt_v;
   print "$msg\n" if -t or $opt_v;

+
+  unless ($opt_d) {
+    foreach my $ld (@logdir) {
+      if (open my $log,">>$ld/cleanup.log") {
+        print {$log} "$isodate $msg\n";
+        close $log;
+      }
+    }
+  }

 }
 
 
 }
 
 


@@ -585,3 +597,9 @@ sub verbose {
     }
   }
 }
     }
   }
 }

+
+
+sub lmtime {
+  my @s = lstat(shift);
+  return @s?$s[9]:0;
+}