From b346236789181e7a72a21f0cb9f938f3af3c4d0e Mon Sep 17 00:00:00 2001
From: Eric van der Vlist <vdv@dyomedea.com>
Date: Sun, 22 Apr 2012 18:56:15 +0200
Subject: [PATCH] Adding a mechanism to delay actions in the queue.

---
 .../actions/heritrix-archive-set.xpl          | 168 +++++++++---------
 archiver/pipelines/scheduler.xpl              |   4 +-
 2 files changed, 85 insertions(+), 87 deletions(-)

diff --git a/archiver/pipelines/actions/heritrix-archive-set.xpl b/archiver/pipelines/actions/heritrix-archive-set.xpl
index db78ab8..799c167 100644
--- a/archiver/pipelines/actions/heritrix-archive-set.xpl
+++ b/archiver/pipelines/actions/heritrix-archive-set.xpl
@@ -6,92 +6,11 @@
 -->
 
 <p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xforms="http://www.w3.org/2002/xforms"
-    xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:saxon="http://saxon.sf.net/"
+    xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:saxon="http://saxon.sf.net/"
     xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary">
 
     <p:param name="data" type="input"/>
 
-    <p:processor name="oxf:unsafe-xslt">
-        <p:input name="data" href="#data"/>
-        <p:input name="config">
-            <config xsl:version="2.0">
-                <relpath>queue.xml</relpath>
-                <operation>write</operation>
-                <type>xquery</type>
-                <parameter name="directory" type="string">
-                    <xsl:value-of select="translate(/action/@uuid, '-', '/')"/>
-                    <xsl:text>/</xsl:text>
-                </parameter>
-                <parameter name="filename" type="string">
-                    <xsl:value-of select="saxon:string-to-hexBinary(/action/@url, 'utf-8')"/>
-                    <xsl:text>.xml</xsl:text>
-                </parameter>
-                <parameter name="uuid" type="string">
-                    <xsl:value-of select="/action/@uuid"/>
-                </parameter>
-                <parameter name="url" type="string">
-                    <xsl:value-of select="/action/@url"/>
-                </parameter>
-                <parameter name="priority-resource" type="string">
-                    <xsl:value-of select="/action/@priority + 2"/>
-                </parameter>
-                <parameter name="priority-package" type="string">
-                    <xsl:value-of select="/action/@priority + 1"/>
-                </parameter>
-            </config>
-        </p:input>
-        <p:output name="data" id="data-access-data"/>
-    </p:processor>
-
-    <!--    <p:processor name="oxf:pipeline">
-        <p:input name="config" href="/data-access.xpl"/>
-        <p:input name="data" href="#data-access-data"/>
-        <p:input name="param">
-            <xquery><![CDATA[
-declare namespace util = "http://exist-db.org/xquery/util";
-
-for $q in /queue return
-    update 
-        insert (<action priority=$(priority-resource) uuid="{util:uuid()}" type="archive-resource" url=$(url) directory=$(directory) filename=$(filename)/>,
-               <action priority=$(priority-package) uuid="{util:uuid()}" type="package-archive" directory=$(directory)/>)
-        into $q,
-        
-for $a in /queue/action where $a/@uuid = $(uuid) return
-    update
-        delete $a
-        
-                ]]></xquery>
-        </p:input>
-        <p:output name="data" id="response" debug="response"/>
-    </p:processor>
-
-    <p:processor name="oxf:null-serializer">
-        <p:input name="data" href="#response"/>
-    </p:processor>
--->
-    <p:processor name="oxf:pipeline">
-        <p:input name="config" href="/data-access.xpl"/>
-        <p:input name="data" transform="oxf:xslt" href="#data-access-data">
-            <config xsl:version="2.0">
-                <relpath>
-                    <xsl:value-of select="/config/parameter[@name='directory']"/>
-                    <xsl:text>index.xml</xsl:text>
-                </relpath>
-                <operation>write</operation>
-                <type>document</type>
-            </config>
-        </p:input>
-        <p:input name="param" transform="oxf:xslt" href="#data">
-            <archive-set xsl:version="2.0">
-                <xsl:copy-of select="/action/@url|/action/@uuid"/>
-            </archive-set>
-        </p:input>
-        <p:output name="data" id="response2" debug="response2"/>
-    </p:processor>
-
-    <p:processor name="oxf:null-serializer">
-        <p:input name="data" href="#response2"/>
-    </p:processor>
 
     <!-- Create a new Heritrix job-->
     <p:processor name="oxf:xforms-submission">
@@ -151,7 +70,7 @@ for $a in /queue/action where $a/@uuid = $(uuid) return
         </p:input>
         <p:output name="response" id="heritrix-built" debug="heritrix-built"/>
     </p:processor>
-    
+
     <!-- Launch the job -->
     <p:processor name="oxf:xforms-submission">
         <p:input name="submission" transform="oxf:xslt" href="aggregate('root', #data, #heritrix-engine, #heritrix-built)">
@@ -170,7 +89,7 @@ for $a in /queue/action where $a/@uuid = $(uuid) return
         </p:input>
         <p:output name="response" id="heritrix-launched" debug="heritrix-launched"/>
     </p:processor>
-    
+
     <!-- Unpause the job -->
     <p:processor name="oxf:xforms-submission">
         <p:input name="submission" transform="oxf:xslt" href="aggregate('root', #data, #heritrix-engine, #heritrix-launched)">
@@ -189,10 +108,89 @@ for $a in /queue/action where $a/@uuid = $(uuid) return
         </p:input>
         <p:output name="response" id="heritrix-unpaused" debug="heritrix-unpaused"/>
     </p:processor>
+
+
+    <p:processor name="oxf:unsafe-xslt">
+        <p:input name="data" href="aggregate('root', #data, #heritrix-engine, #heritrix-unpaused)"/>
+        <p:input name="config">
+            <config xsl:version="2.0">
+                <relpath>queue.xml</relpath>
+                <operation>write</operation>
+                <type>xquery</type>
+                <parameter name="directory" type="string">
+                    <xsl:value-of select="translate(/root/action/@uuid, '-', '/')"/>
+                    <xsl:text>/</xsl:text>
+                </parameter>
+                <parameter name="uuid" type="string">
+                    <xsl:value-of select="/root/action/@uuid"/>
+                </parameter>
+                <parameter name="url" type="string">
+                    <xsl:value-of select="/root/action/@url"/>
+                </parameter>
+                <parameter name="priority-warc" type="string">
+                    <xsl:value-of select="/root/action/@priority + 1"/>
+                </parameter>
+                <parameter name="next-time" type="string">
+                    <xsl:value-of select="current-dateTime() +  xs:dayTimeDuration('PT1M')"/>
+                </parameter>
+                <parameter name="heritrix-job-url" type="string">
+                    <xsl:value-of select="/root/engine/jobs/value[shortName=/root/action/@uuid]/url"/>
+                </parameter>
+            </config>
+        </p:input>
+        <p:output name="data" id="data-access-data"/>
+    </p:processor>
     
     
+    <p:processor name="oxf:pipeline">
+        <p:input name="config" href="/data-access.xpl"/>
+        <p:input name="data" transform="oxf:xslt" href="#data-access-data">
+            <config xsl:version="2.0">
+                <relpath>
+                    <xsl:value-of select="/config/parameter[@name='directory']"/>
+                    <xsl:text>index.xml</xsl:text>
+                </relpath>
+                <operation>write</operation>
+                <type>document</type>
+            </config>
+        </p:input>
+        <p:input name="param" transform="oxf:xslt" href="#data-access-data">
+            <archive-set xsl:version="2.0" url="{/config/parameter[@name='url']}" uuid="{/config/parameter[@name='uuid']}">
+                <heritrix-job url="{/config/parameter[@name='heritrix-job-url']}"/>
+            </archive-set>
+        </p:input>
+        <p:output name="data" id="response2" debug="response2"/>
+    </p:processor>
+    
     <p:processor name="oxf:null-serializer">
-        <p:input name="data" href="#heritrix-unpaused"/>
+        <p:input name="data" href="#response2"/>
+    </p:processor>
+    
+
+    <p:processor name="oxf:pipeline">
+        <p:input name="config" href="/data-access.xpl"/>
+        <p:input name="data" href="#data-access-data"/>
+        <p:input name="param">
+            <xquery><![CDATA[
+declare namespace util = "http://exist-db.org/xquery/util";
+
+for $q in /queue return
+    update 
+        insert <action priority=$(priority-warc) uuid="{util:uuid()}" type="get-heritrix-warc" url=$(url) directory=$(directory) heritrix-job-url=$(heritrix-job-url) after=$(next-time)/>
+        into $q,
+        
+for $a in /queue/action where $a/@uuid = $(uuid) return
+    update
+        delete $a
+        
+                ]]></xquery>
+        </p:input>
+        <p:output name="data" id="response" debug="response"/>
     </p:processor>
 
+    <p:processor name="oxf:null-serializer">
+        <p:input name="data" href="#response"/>
+    </p:processor>
+
+
 </p:config>
diff --git a/archiver/pipelines/scheduler.xpl b/archiver/pipelines/scheduler.xpl
index fb02905..24ac531 100644
--- a/archiver/pipelines/scheduler.xpl
+++ b/archiver/pipelines/scheduler.xpl
@@ -6,7 +6,7 @@
 -->
 
 <p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xforms="http://www.w3.org/2002/xforms"
-    xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary">
+    xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary">
 
     <p:processor name="oxf:pipeline">
         <p:input name="config" href="data-access.xpl"/>
@@ -20,7 +20,7 @@
         <p:input name="param">
             <xquery><![CDATA[
 
-/queue/action[@priority=max(/queue/action/@priority)]
+/queue/action[not(@after) or xs:dateTime(@after) < current-dateTime()][@priority=max(/queue/action[not(@after) or xs:dateTime(@after) < current-dateTime()]/@priority)]
                 
                 ]]></xquery>
         </p:input>