Adding a mechanism to delay actions in the queue.
This commit is contained in:
parent
3bcb813cb7
commit
b346236789
|
|
@ -6,92 +6,11 @@
|
||||||
-->
|
-->
|
||||||
|
|
||||||
<p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xforms="http://www.w3.org/2002/xforms"
|
<p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xforms="http://www.w3.org/2002/xforms"
|
||||||
xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:saxon="http://saxon.sf.net/"
|
xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:saxon="http://saxon.sf.net/"
|
||||||
xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary">
|
xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary">
|
||||||
|
|
||||||
<p:param name="data" type="input"/>
|
<p:param name="data" type="input"/>
|
||||||
|
|
||||||
<p:processor name="oxf:unsafe-xslt">
|
|
||||||
<p:input name="data" href="#data"/>
|
|
||||||
<p:input name="config">
|
|
||||||
<config xsl:version="2.0">
|
|
||||||
<relpath>queue.xml</relpath>
|
|
||||||
<operation>write</operation>
|
|
||||||
<type>xquery</type>
|
|
||||||
<parameter name="directory" type="string">
|
|
||||||
<xsl:value-of select="translate(/action/@uuid, '-', '/')"/>
|
|
||||||
<xsl:text>/</xsl:text>
|
|
||||||
</parameter>
|
|
||||||
<parameter name="filename" type="string">
|
|
||||||
<xsl:value-of select="saxon:string-to-hexBinary(/action/@url, 'utf-8')"/>
|
|
||||||
<xsl:text>.xml</xsl:text>
|
|
||||||
</parameter>
|
|
||||||
<parameter name="uuid" type="string">
|
|
||||||
<xsl:value-of select="/action/@uuid"/>
|
|
||||||
</parameter>
|
|
||||||
<parameter name="url" type="string">
|
|
||||||
<xsl:value-of select="/action/@url"/>
|
|
||||||
</parameter>
|
|
||||||
<parameter name="priority-resource" type="string">
|
|
||||||
<xsl:value-of select="/action/@priority + 2"/>
|
|
||||||
</parameter>
|
|
||||||
<parameter name="priority-package" type="string">
|
|
||||||
<xsl:value-of select="/action/@priority + 1"/>
|
|
||||||
</parameter>
|
|
||||||
</config>
|
|
||||||
</p:input>
|
|
||||||
<p:output name="data" id="data-access-data"/>
|
|
||||||
</p:processor>
|
|
||||||
|
|
||||||
<!-- <p:processor name="oxf:pipeline">
|
|
||||||
<p:input name="config" href="/data-access.xpl"/>
|
|
||||||
<p:input name="data" href="#data-access-data"/>
|
|
||||||
<p:input name="param">
|
|
||||||
<xquery><![CDATA[
|
|
||||||
declare namespace util = "http://exist-db.org/xquery/util";
|
|
||||||
|
|
||||||
for $q in /queue return
|
|
||||||
update
|
|
||||||
insert (<action priority=$(priority-resource) uuid="{util:uuid()}" type="archive-resource" url=$(url) directory=$(directory) filename=$(filename)/>,
|
|
||||||
<action priority=$(priority-package) uuid="{util:uuid()}" type="package-archive" directory=$(directory)/>)
|
|
||||||
into $q,
|
|
||||||
|
|
||||||
for $a in /queue/action where $a/@uuid = $(uuid) return
|
|
||||||
update
|
|
||||||
delete $a
|
|
||||||
|
|
||||||
]]></xquery>
|
|
||||||
</p:input>
|
|
||||||
<p:output name="data" id="response" debug="response"/>
|
|
||||||
</p:processor>
|
|
||||||
|
|
||||||
<p:processor name="oxf:null-serializer">
|
|
||||||
<p:input name="data" href="#response"/>
|
|
||||||
</p:processor>
|
|
||||||
-->
|
|
||||||
<p:processor name="oxf:pipeline">
|
|
||||||
<p:input name="config" href="/data-access.xpl"/>
|
|
||||||
<p:input name="data" transform="oxf:xslt" href="#data-access-data">
|
|
||||||
<config xsl:version="2.0">
|
|
||||||
<relpath>
|
|
||||||
<xsl:value-of select="/config/parameter[@name='directory']"/>
|
|
||||||
<xsl:text>index.xml</xsl:text>
|
|
||||||
</relpath>
|
|
||||||
<operation>write</operation>
|
|
||||||
<type>document</type>
|
|
||||||
</config>
|
|
||||||
</p:input>
|
|
||||||
<p:input name="param" transform="oxf:xslt" href="#data">
|
|
||||||
<archive-set xsl:version="2.0">
|
|
||||||
<xsl:copy-of select="/action/@url|/action/@uuid"/>
|
|
||||||
</archive-set>
|
|
||||||
</p:input>
|
|
||||||
<p:output name="data" id="response2" debug="response2"/>
|
|
||||||
</p:processor>
|
|
||||||
|
|
||||||
<p:processor name="oxf:null-serializer">
|
|
||||||
<p:input name="data" href="#response2"/>
|
|
||||||
</p:processor>
|
|
||||||
|
|
||||||
<!-- Create a new Heritrix job-->
|
<!-- Create a new Heritrix job-->
|
||||||
<p:processor name="oxf:xforms-submission">
|
<p:processor name="oxf:xforms-submission">
|
||||||
|
|
@ -151,7 +70,7 @@ for $a in /queue/action where $a/@uuid = $(uuid) return
|
||||||
</p:input>
|
</p:input>
|
||||||
<p:output name="response" id="heritrix-built" debug="heritrix-built"/>
|
<p:output name="response" id="heritrix-built" debug="heritrix-built"/>
|
||||||
</p:processor>
|
</p:processor>
|
||||||
|
|
||||||
<!-- Launch the job -->
|
<!-- Launch the job -->
|
||||||
<p:processor name="oxf:xforms-submission">
|
<p:processor name="oxf:xforms-submission">
|
||||||
<p:input name="submission" transform="oxf:xslt" href="aggregate('root', #data, #heritrix-engine, #heritrix-built)">
|
<p:input name="submission" transform="oxf:xslt" href="aggregate('root', #data, #heritrix-engine, #heritrix-built)">
|
||||||
|
|
@ -170,7 +89,7 @@ for $a in /queue/action where $a/@uuid = $(uuid) return
|
||||||
</p:input>
|
</p:input>
|
||||||
<p:output name="response" id="heritrix-launched" debug="heritrix-launched"/>
|
<p:output name="response" id="heritrix-launched" debug="heritrix-launched"/>
|
||||||
</p:processor>
|
</p:processor>
|
||||||
|
|
||||||
<!-- Unpause the job -->
|
<!-- Unpause the job -->
|
||||||
<p:processor name="oxf:xforms-submission">
|
<p:processor name="oxf:xforms-submission">
|
||||||
<p:input name="submission" transform="oxf:xslt" href="aggregate('root', #data, #heritrix-engine, #heritrix-launched)">
|
<p:input name="submission" transform="oxf:xslt" href="aggregate('root', #data, #heritrix-engine, #heritrix-launched)">
|
||||||
|
|
@ -189,10 +108,89 @@ for $a in /queue/action where $a/@uuid = $(uuid) return
|
||||||
</p:input>
|
</p:input>
|
||||||
<p:output name="response" id="heritrix-unpaused" debug="heritrix-unpaused"/>
|
<p:output name="response" id="heritrix-unpaused" debug="heritrix-unpaused"/>
|
||||||
</p:processor>
|
</p:processor>
|
||||||
|
|
||||||
|
|
||||||
|
<p:processor name="oxf:unsafe-xslt">
|
||||||
|
<p:input name="data" href="aggregate('root', #data, #heritrix-engine, #heritrix-unpaused)"/>
|
||||||
|
<p:input name="config">
|
||||||
|
<config xsl:version="2.0">
|
||||||
|
<relpath>queue.xml</relpath>
|
||||||
|
<operation>write</operation>
|
||||||
|
<type>xquery</type>
|
||||||
|
<parameter name="directory" type="string">
|
||||||
|
<xsl:value-of select="translate(/root/action/@uuid, '-', '/')"/>
|
||||||
|
<xsl:text>/</xsl:text>
|
||||||
|
</parameter>
|
||||||
|
<parameter name="uuid" type="string">
|
||||||
|
<xsl:value-of select="/root/action/@uuid"/>
|
||||||
|
</parameter>
|
||||||
|
<parameter name="url" type="string">
|
||||||
|
<xsl:value-of select="/root/action/@url"/>
|
||||||
|
</parameter>
|
||||||
|
<parameter name="priority-warc" type="string">
|
||||||
|
<xsl:value-of select="/root/action/@priority + 1"/>
|
||||||
|
</parameter>
|
||||||
|
<parameter name="next-time" type="string">
|
||||||
|
<xsl:value-of select="current-dateTime() + xs:dayTimeDuration('PT1M')"/>
|
||||||
|
</parameter>
|
||||||
|
<parameter name="heritrix-job-url" type="string">
|
||||||
|
<xsl:value-of select="/root/engine/jobs/value[shortName=/root/action/@uuid]/url"/>
|
||||||
|
</parameter>
|
||||||
|
</config>
|
||||||
|
</p:input>
|
||||||
|
<p:output name="data" id="data-access-data"/>
|
||||||
|
</p:processor>
|
||||||
|
|
||||||
|
|
||||||
|
<p:processor name="oxf:pipeline">
|
||||||
|
<p:input name="config" href="/data-access.xpl"/>
|
||||||
|
<p:input name="data" transform="oxf:xslt" href="#data-access-data">
|
||||||
|
<config xsl:version="2.0">
|
||||||
|
<relpath>
|
||||||
|
<xsl:value-of select="/config/parameter[@name='directory']"/>
|
||||||
|
<xsl:text>index.xml</xsl:text>
|
||||||
|
</relpath>
|
||||||
|
<operation>write</operation>
|
||||||
|
<type>document</type>
|
||||||
|
</config>
|
||||||
|
</p:input>
|
||||||
|
<p:input name="param" transform="oxf:xslt" href="#data-access-data">
|
||||||
|
<archive-set xsl:version="2.0" url="{/config/parameter[@name='url']}" uuid="{/config/parameter[@name='uuid']}">
|
||||||
|
<heritrix-job url="{/config/parameter[@name='heritrix-job-url']}"/>
|
||||||
|
</archive-set>
|
||||||
|
</p:input>
|
||||||
|
<p:output name="data" id="response2" debug="response2"/>
|
||||||
|
</p:processor>
|
||||||
|
|
||||||
<p:processor name="oxf:null-serializer">
|
<p:processor name="oxf:null-serializer">
|
||||||
<p:input name="data" href="#heritrix-unpaused"/>
|
<p:input name="data" href="#response2"/>
|
||||||
|
</p:processor>
|
||||||
|
|
||||||
|
|
||||||
|
<p:processor name="oxf:pipeline">
|
||||||
|
<p:input name="config" href="/data-access.xpl"/>
|
||||||
|
<p:input name="data" href="#data-access-data"/>
|
||||||
|
<p:input name="param">
|
||||||
|
<xquery><![CDATA[
|
||||||
|
declare namespace util = "http://exist-db.org/xquery/util";
|
||||||
|
|
||||||
|
for $q in /queue return
|
||||||
|
update
|
||||||
|
insert <action priority=$(priority-warc) uuid="{util:uuid()}" type="get-heritrix-warc" url=$(url) directory=$(directory) heritrix-job-url=$(heritrix-job-url) after=$(next-time)/>
|
||||||
|
into $q,
|
||||||
|
|
||||||
|
for $a in /queue/action where $a/@uuid = $(uuid) return
|
||||||
|
update
|
||||||
|
delete $a
|
||||||
|
|
||||||
|
]]></xquery>
|
||||||
|
</p:input>
|
||||||
|
<p:output name="data" id="response" debug="response"/>
|
||||||
</p:processor>
|
</p:processor>
|
||||||
|
|
||||||
|
<p:processor name="oxf:null-serializer">
|
||||||
|
<p:input name="data" href="#response"/>
|
||||||
|
</p:processor>
|
||||||
|
|
||||||
|
|
||||||
</p:config>
|
</p:config>
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@
|
||||||
-->
|
-->
|
||||||
|
|
||||||
<p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xforms="http://www.w3.org/2002/xforms"
|
<p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xforms="http://www.w3.org/2002/xforms"
|
||||||
xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary">
|
xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary">
|
||||||
|
|
||||||
<p:processor name="oxf:pipeline">
|
<p:processor name="oxf:pipeline">
|
||||||
<p:input name="config" href="data-access.xpl"/>
|
<p:input name="config" href="data-access.xpl"/>
|
||||||
|
|
@ -20,7 +20,7 @@
|
||||||
<p:input name="param">
|
<p:input name="param">
|
||||||
<xquery><![CDATA[
|
<xquery><![CDATA[
|
||||||
|
|
||||||
/queue/action[@priority=max(/queue/action/@priority)]
|
/queue/action[not(@after) or xs:dateTime(@after) < current-dateTime()][@priority=max(/queue/action[not(@after) or xs:dateTime(@after) < current-dateTime()]/@priority)]
|
||||||
|
|
||||||
]]></xquery>
|
]]></xquery>
|
||||||
</p:input>
|
</p:input>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue