From 3d18e9d8a4f2a0c75a8cd60f1ed648bc537505b7 Mon Sep 17 00:00:00 2001
From: Eric van der Vlist <vdv@dyomedea.com>
Date: Fri, 13 Apr 2012 13:05:25 +0200
Subject: [PATCH] Adding a mechanism to avoid to archive multiple times the
 same resource for a single archive set.

---
 .../pipelines/actions/archive-resource.xpl    | 403 ++++++++++--------
 1 file changed, 234 insertions(+), 169 deletions(-)

diff --git a/archiver/pipelines/actions/archive-resource.xpl b/archiver/pipelines/actions/archive-resource.xpl
index 85e014c..798dc67 100644
--- a/archiver/pipelines/actions/archive-resource.xpl
+++ b/archiver/pipelines/actions/archive-resource.xpl
@@ -2,161 +2,222 @@
 
   <p:param name="data" type="input"/>
 
-  <!-- Fetch the resource -->
-  <p:processor name="oxf:url-generator">
-    <p:input name="config" transform="oxf:xslt" href="#data">
-      <config xsl:version="2.0">
-        <url>
-          <xsl:value-of select="/action/@url"/>
-        </url>
-        <header>
-          <name>User-Agent</name>
-          <value>
-            <xsl:value-of select="doc('oxf:/config.xml')/config/user-agent"/>
-          </value>
-        </header>
-        <mode>archive</mode>
-      </config>
-    </p:input>
-    <p:output name="data" id="archive" debug="archive"/>
-  </p:processor>
-
-
-  <!-- Store the archive in the database -->
+  <!-- Look if the resource has already been archived for that set -->
   <p:processor name="oxf:pipeline">
     <p:input name="config" href="/data-access.xpl"/>
     <p:input name="data" transform="oxf:xslt" href="#data">
       <config xsl:version="2.0">
         <relpath>
           <xsl:value-of select="/action/@directory"/>
-          <xsl:value-of select="/action/@filename"/>
+          <xsl:text>index.xml</xsl:text>
         </relpath>
-        <operation>write</operation>
-        <type>document</type>
+        <operation>read</operation>
+        <type>xquery</type>
+        <parameter name="url" type="string">
+          <xsl:value-of select="/action/@url"/>
+        </parameter>
       </config>
     </p:input>
-    <p:input name="param" href="#archive"/>
-    <p:output name="data" id="response2"/>
+    <p:input name="param">
+      <xquery><![CDATA[
+        
+boolean(//archive[@url = $(url)])
+                
+                ]]></xquery>
+    </p:input>
+    <p:output name="data" id="duplicate" debug="duplicate"/>
   </p:processor>
 
-  <p:processor name="oxf:null-serializer">
-    <p:input name="data" href="#response2"/>
-  </p:processor>
+  <p:choose href="#duplicate">
 
-
-  <!-- Test the type of document to see if it needs to be rewritten -->
-  <p:choose href="#archive">
-
-    <!-- HTML document : need to update the links... -->
-    <p:when test="/archive/response/document/@content-type=('text/html', 'text/css')">
-
-      <!-- Call the corresponding pipeline to extract the links and rewrite them -->
-      <p:processor name="oxf:url-generator">
-        <p:input name="config" transform="oxf:xslt" href="#archive">
+    <p:when test="/*/* = 'true'">
+      <!-- Already archived, nothing to do -->
+      <!-- Update the queue -->
+      <p:processor name="oxf:pipeline">
+        <p:input name="config" href="/data-access.xpl"/>
+        <p:input name="data" transform="oxf:xslt" href="#data">
           <config xsl:version="2.0">
-            <url>
-              <xsl:text>oxf:/actions/mediatypes/</xsl:text>
-              <xsl:value-of select="substring-after(/archive/response/document/@content-type, '/')"/>
-              <xsl:text>.xpl</xsl:text>
-            </url>
+            <relpath>queue.xml</relpath>
+            <operation>write</operation>
+            <type>xquery</type>
+            <parameter name="uuid" type="string">
+              <xsl:value-of select="/action/@uuid"/>
+            </parameter>
           </config>
         </p:input>
-        <p:output name="data" id="pipeline"/>
+        <p:input name="param">
+          <xquery><![CDATA[
+            
+for $a in /queue/action where $a/@uuid = $(uuid) return
+    update
+        delete $a
+        
+                ]]></xquery>
+        </p:input>
+        <p:output name="data" id="response4" debug="response"/>
       </p:processor>
+      <p:processor name="oxf:null-serializer">
+        <p:input name="data" href="#response4"/>
+      </p:processor>
+    </p:when>
 
-      <p:processor name="oxf:pipeline">
-        <p:input name="config" href="#pipeline"/>
-        <p:input name="archive" href="#archive"/>
-        <p:output name="rewritten" id="rewritten"/>
-        <p:output name="links" id="links"/>
+    <p:otherwise>
+      <!-- Otherwise, archive the resource... -->
+      <!-- Fetch the resource -->
+      <p:processor name="oxf:url-generator">
+        <p:input name="config" transform="oxf:xslt" href="#data">
+          <config xsl:version="2.0">
+            <url>
+              <xsl:value-of select="/action/@url"/>
+            </url>
+            <header>
+              <name>User-Agent</name>
+              <value>
+                <xsl:value-of select="doc('oxf:/config.xml')/config/user-agent"/>
+              </value>
+            </header>
+            <mode>archive</mode>
+          </config>
+        </p:input>
+        <p:output name="data" id="archive" debug="archive"/>
       </p:processor>
 
 
-      <!-- Store the rewritten document in the database -->
+      <!-- Store the archive in the database -->
       <p:processor name="oxf:pipeline">
         <p:input name="config" href="/data-access.xpl"/>
         <p:input name="data" transform="oxf:xslt" href="#data">
           <config xsl:version="2.0">
             <relpath>
               <xsl:value-of select="/action/@directory"/>
-              <xsl:text>rewritten-</xsl:text>
               <xsl:value-of select="/action/@filename"/>
             </relpath>
             <operation>write</operation>
             <type>document</type>
           </config>
         </p:input>
-        <p:input name="param" href="#rewritten"/>
-        <p:output name="data" id="response3"/>
+        <p:input name="param" href="#archive"/>
+        <p:output name="data" id="response2"/>
       </p:processor>
+
       <p:processor name="oxf:null-serializer">
-        <p:input name="data" href="#response3"/>
+        <p:input name="data" href="#response2"/>
       </p:processor>
 
 
+      <!-- Test the type of document to see if it needs to be rewritten -->
+      <p:choose href="#archive">
 
-      <!-- Update the archive index -->
-      <p:processor name="oxf:pipeline">
-        <p:input name="config" href="/data-access.xpl"/>
-        <p:input name="data" transform="oxf:xslt" href="#data">
-          <config xsl:version="2.0">
-            <relpath>
-              <xsl:value-of select="/action/@directory"/>
-              <xsl:text>index.xml</xsl:text>
-            </relpath>
-            <operation>write</operation>
-            <type>xquery</type>
-            <parameter name="url" type="string">
-              <xsl:value-of select="/action/@url"/>
-            </parameter>
-            <parameter name="filename" type="string">
-              <xsl:value-of select="/action/@filename"/>
-            </parameter>
-            <parameter name="filename-rewritten" type="string">
-              <xsl:text>rewritten-</xsl:text>
-              <xsl:value-of select="/action/@filename"/>
-            </parameter>
-          </config>
-        </p:input>
-        <p:input name="param">
-          <xquery><![CDATA[
+        <!-- HTML document : need to update the links... -->
+        <p:when test="/archive/response/document/@content-type=('text/html', 'text/css')">
+
+          <!-- Call the corresponding pipeline to extract the links and rewrite them -->
+          <p:processor name="oxf:url-generator">
+            <p:input name="config" transform="oxf:xslt" href="#archive">
+              <config xsl:version="2.0">
+                <url>
+                  <xsl:text>oxf:/actions/mediatypes/</xsl:text>
+                  <xsl:value-of select="substring-after(/archive/response/document/@content-type, '/')"/>
+                  <xsl:text>.xpl</xsl:text>
+                </url>
+              </config>
+            </p:input>
+            <p:output name="data" id="pipeline"/>
+          </p:processor>
+
+          <p:processor name="oxf:pipeline">
+            <p:input name="config" href="#pipeline"/>
+            <p:input name="archive" href="#archive"/>
+            <p:output name="rewritten" id="rewritten"/>
+            <p:output name="links" id="links"/>
+          </p:processor>
+
+
+          <!-- Store the rewritten document in the database -->
+          <p:processor name="oxf:pipeline">
+            <p:input name="config" href="/data-access.xpl"/>
+            <p:input name="data" transform="oxf:xslt" href="#data">
+              <config xsl:version="2.0">
+                <relpath>
+                  <xsl:value-of select="/action/@directory"/>
+                  <xsl:text>rewritten-</xsl:text>
+                  <xsl:value-of select="/action/@filename"/>
+                </relpath>
+                <operation>write</operation>
+                <type>document</type>
+              </config>
+            </p:input>
+            <p:input name="param" href="#rewritten"/>
+            <p:output name="data" id="response3"/>
+          </p:processor>
+          <p:processor name="oxf:null-serializer">
+            <p:input name="data" href="#response3"/>
+          </p:processor>
+
+
+
+          <!-- Update the archive index -->
+          <p:processor name="oxf:pipeline">
+            <p:input name="config" href="/data-access.xpl"/>
+            <p:input name="data" transform="oxf:xslt" href="#data">
+              <config xsl:version="2.0">
+                <relpath>
+                  <xsl:value-of select="/action/@directory"/>
+                  <xsl:text>index.xml</xsl:text>
+                </relpath>
+                <operation>write</operation>
+                <type>xquery</type>
+                <parameter name="url" type="string">
+                  <xsl:value-of select="/action/@url"/>
+                </parameter>
+                <parameter name="filename" type="string">
+                  <xsl:value-of select="/action/@filename"/>
+                </parameter>
+                <parameter name="filename-rewritten" type="string">
+                  <xsl:text>rewritten-</xsl:text>
+                  <xsl:value-of select="/action/@filename"/>
+                </parameter>
+              </config>
+            </p:input>
+            <p:input name="param">
+              <xquery><![CDATA[
 for $as in /archive-set 
     return 
       update 
         insert <archive url=$(url) href=$(filename) href-rewritten=$(filename-rewritten) dateTime="{current-dateTime()}"/> 
         into $as                
                 ]]></xquery>
-        </p:input>
-        <p:output name="data" id="response1"/>
-      </p:processor>
-      <p:processor name="oxf:null-serializer">
-        <p:input name="data" href="#response1"/>
-      </p:processor>
+            </p:input>
+            <p:output name="data" id="response1"/>
+          </p:processor>
+          <p:processor name="oxf:null-serializer">
+            <p:input name="data" href="#response1"/>
+          </p:processor>
 
-      <!-- Update the queue -->
-      <p:processor name="oxf:pipeline">
-        <p:input name="config" href="/data-access.xpl"/>
-        <p:input name="data" transform="oxf:xslt" href="aggregate('root', #data, #links)">
-          <config xsl:version="2.0">
-            <relpath>queue.xml</relpath>
-            <operation>write</operation>
-            <type>xquery</type>
-            <parameter name="directory" type="string">
-              <xsl:value-of select="/root/action/@directory"/>
-            </parameter>
-            <parameter name="uuid" type="string">
-              <xsl:value-of select="/root/action/@uuid"/>
-            </parameter>
-            <parameter name="priority" type="string">
-              <xsl:value-of select="/root/action/@priority"/>
-            </parameter>
-            <parameter name="links" type="node-set">
-              <xsl:copy-of select="/root/links"/>
-            </parameter>
-          </config>
-        </p:input>
-        <p:input name="param">
-          <xquery><![CDATA[
+          <!-- Update the queue -->
+          <p:processor name="oxf:pipeline">
+            <p:input name="config" href="/data-access.xpl"/>
+            <p:input name="data" transform="oxf:xslt" href="aggregate('root', #data, #links)">
+              <config xsl:version="2.0">
+                <relpath>queue.xml</relpath>
+                <operation>write</operation>
+                <type>xquery</type>
+                <parameter name="directory" type="string">
+                  <xsl:value-of select="/root/action/@directory"/>
+                </parameter>
+                <parameter name="uuid" type="string">
+                  <xsl:value-of select="/root/action/@uuid"/>
+                </parameter>
+                <parameter name="priority" type="string">
+                  <xsl:value-of select="/root/action/@priority"/>
+                </parameter>
+                <parameter name="links" type="node-set">
+                  <xsl:copy-of select="/root/links"/>
+                </parameter>
+              </config>
+            </p:input>
+            <p:input name="param">
+              <xquery><![CDATA[
 declare namespace util = "http://exist-db.org/xquery/util";
 declare variable $links := $(links);
 
@@ -174,82 +235,86 @@ for $a in /queue/action where $a/@uuid = $(uuid) return
         delete $a
         
                 ]]></xquery>
-        </p:input>
-        <p:output name="data" id="response4" debug="response"/>
-      </p:processor>
-      <p:processor name="oxf:null-serializer">
-        <p:input name="data" href="#response4"/>
-      </p:processor>
+            </p:input>
+            <p:output name="data" id="response4" debug="response"/>
+          </p:processor>
+          <p:processor name="oxf:null-serializer">
+            <p:input name="data" href="#response4"/>
+          </p:processor>
 
 
-    </p:when>
+        </p:when>
 
-    <!-- Otherwise: no need to rewrite -->
-    <p:otherwise>
-      <!-- Update the archive index -->
-      <p:processor name="oxf:pipeline">
-        <p:input name="config" href="/data-access.xpl"/>
-        <p:input name="data" transform="oxf:xslt" href="#data">
-          <config xsl:version="2.0">
-            <relpath>
-              <xsl:value-of select="/action/@directory"/>
-              <xsl:text>index.xml</xsl:text>
-            </relpath>
-            <operation>write</operation>
-            <type>xquery</type>
-            <parameter name="url" type="string">
-              <xsl:value-of select="/action/@url"/>
-            </parameter>
-            <parameter name="filename" type="string">
-              <xsl:value-of select="/action/@filename"/>
-            </parameter>
-          </config>
-        </p:input>
-        <p:input name="param">
-          <xquery><![CDATA[
+        <!-- Otherwise: no need to rewrite -->
+        <p:otherwise>
+          <!-- Update the archive index -->
+          <p:processor name="oxf:pipeline">
+            <p:input name="config" href="/data-access.xpl"/>
+            <p:input name="data" transform="oxf:xslt" href="#data">
+              <config xsl:version="2.0">
+                <relpath>
+                  <xsl:value-of select="/action/@directory"/>
+                  <xsl:text>index.xml</xsl:text>
+                </relpath>
+                <operation>write</operation>
+                <type>xquery</type>
+                <parameter name="url" type="string">
+                  <xsl:value-of select="/action/@url"/>
+                </parameter>
+                <parameter name="filename" type="string">
+                  <xsl:value-of select="/action/@filename"/>
+                </parameter>
+              </config>
+            </p:input>
+            <p:input name="param">
+              <xquery><![CDATA[
 for $as in /archive-set 
     return 
       update 
         insert <archive url=$(url) href=$(filename) dateTime="{current-dateTime()}"/> 
         into $as                
                 ]]></xquery>
-        </p:input>
-        <p:output name="data" id="response1"/>
-      </p:processor>
-      <p:processor name="oxf:null-serializer">
-        <p:input name="data" href="#response1"/>
-      </p:processor>
-
-      <!-- Update the queue -->
-      <p:processor name="oxf:pipeline">
-        <p:input name="config" href="/data-access.xpl"/>
-        <p:input name="data" transform="oxf:xslt" href="#data">
-          <config xsl:version="2.0">
-            <relpath>queue.xml</relpath>
-            <operation>write</operation>
-            <type>xquery</type>
-            <parameter name="uuid" type="string">
-              <xsl:value-of select="/action/@uuid"/>
-            </parameter>
-          </config>
-        </p:input>
-        <p:input name="param">
-          <xquery><![CDATA[
+            </p:input>
+            <p:output name="data" id="response1"/>
+          </p:processor>
+          <p:processor name="oxf:null-serializer">
+            <p:input name="data" href="#response1"/>
+          </p:processor>
 
+          <!-- Update the queue -->
+          <p:processor name="oxf:pipeline">
+            <p:input name="config" href="/data-access.xpl"/>
+            <p:input name="data" transform="oxf:xslt" href="#data">
+              <config xsl:version="2.0">
+                <relpath>queue.xml</relpath>
+                <operation>write</operation>
+                <type>xquery</type>
+                <parameter name="uuid" type="string">
+                  <xsl:value-of select="/action/@uuid"/>
+                </parameter>
+              </config>
+            </p:input>
+            <p:input name="param">
+              <xquery><![CDATA[
+                
 for $a in /queue/action where $a/@uuid = $(uuid) return
     update
         delete $a
         
                 ]]></xquery>
-        </p:input>
-        <p:output name="data" id="response4" debug="response"/>
-      </p:processor>
-      <p:processor name="oxf:null-serializer">
-        <p:input name="data" href="#response4"/>
-      </p:processor>
+            </p:input>
+            <p:output name="data" id="response4" debug="response"/>
+          </p:processor>
+          <p:processor name="oxf:null-serializer">
+            <p:input name="data" href="#response4"/>
+          </p:processor>
 
+        </p:otherwise>
+
+      </p:choose>
     </p:otherwise>
-
   </p:choose>
 
+
+
 </p:config>