Dyomedea
/
owark
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Starting to support content lengths in warc archives

This commit is contained in:
Eric van der Vlist 2012-04-14 22:32:33 +02:00
parent 9d99928c60
commit ba51ddfb0b
2 changed files with 151 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
archiver/pipelines/actions/package-archive.xpl
View File

@ -33,7 +33,8 @@
<xsl:import href="warc-lib.xsl"/> <xsl:import href="warc-lib.xsl"/>
<xsl:template match="/"> <xsl:template match="/">
<xsl:variable name="content" as="node()*"> <xsl:variable name="content" as="node()*">
<version/> <record>
<header>
<field> <field>
<name>WARC-Type</name> <name>WARC-Type</name>
<value>warcinfo</value> <value>warcinfo</value>
@ -56,8 +57,8 @@
<name>Content-Type</name> <name>Content-Type</name>
<value>application/warc-fields</value> <value>application/warc-fields</value>
</field> </field>
<!-- TODO: Content-Length: 381 --> </header>
<CRLF/> <block>
<field> <field>
<name>software</name> <name>software</name>
<value>Owark 0.3 http://owark.org</value> <value>Owark 0.3 http://owark.org</value>
@ -66,8 +67,8 @@
<name>format</name> <name>format</name>
<value>WARC file version 0.18</value> <value>WARC file version 0.18</value>
</field> </field>
<CRLF/> </block>
<CRLF/> </record>
<!-- <!--
@ -130,9 +131,10 @@ conformsTo:
<xsl:stylesheet version="2.0"> <xsl:stylesheet version="2.0">
<xsl:import href="warc-lib.xsl"/> <xsl:import href="warc-lib.xsl"/>
<xsl:template match="/"> <xsl:template match="/">
<xsl:variable name="content" as="node()*"> <xsl:variable name="request" as="node()*">
<!-- Request --> <!-- Request -->
<version/> <record>
<header>
<field> <field>
<name>WARC-Type</name> <name>WARC-Type</name>
<value>request</value> <value>request</value>
@ -162,13 +164,16 @@ conformsTo:
<name>Content-Type</name> <name>Content-Type</name>
<value>application/http;msgtype=request</value> <value>application/http;msgtype=request</value>
</field> </field>
<!-- TODO: Content-Length: 381 --> </header>
<CRLF/> <block>
<xsl:apply-templates select="/archive/request" mode="warc"/> <xsl:apply-templates select="/archive/request" mode="warc-http"/>
<CRLF/> </block>
<CRLF/> </record>
</xsl:variable>
<!-- Response --> <!-- Response -->
<version/> <xsl:variable name="response" as="node()*">
<record>
<header>
<field> <field>
<name>WARC-Type</name> <name>WARC-Type</name>
<value>response</value> <value>response</value>
@ -198,14 +203,15 @@ conformsTo:
<name>Content-Type</name> <name>Content-Type</name>
<value>application/http;msgtype=response</value> <value>application/http;msgtype=response</value>
</field> </field>
<!-- TODO: Content-Length: 381 --> </header>
<CRLF/> <block>
<xsl:apply-templates select="/archive/response" mode="warc"/> <xsl:apply-templates select="/archive/response" mode="warc-http"/>
<CRLF/> </block>
</record>
</xsl:variable> </xsl:variable>
<document xsl:version="2.0" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string" content-type="text/plain"> <document xsl:version="2.0" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string" content-type="text/plain">
<xsl:apply-templates select="$content" mode="warc"/> <xsl:apply-templates select="$request" mode="warc"/>
<xsl:apply-templates select="$response" mode="warc"/>
</document> </document>
</xsl:template> </xsl:template>
</xsl:stylesheet> </xsl:stylesheet>

45
archiver/pipelines/actions/warc-lib.xsl
View File

@ -11,10 +11,10 @@
<xsl:variable name="CRLF" select="'&#13;&#10;'"/> <xsl:variable name="CRLF" select="'&#13;&#10;'"/>
<xsl:variable name="version">WARC/0.18</xsl:variable> <xsl:variable name="version">WARC/0.18</xsl:variable>
<xsl:template match="CRLF" mode="warc"> <xsl:template match="CRLF" name="CRLF" mode="warc">
<xsl:value-of select="$CRLF"/> <xsl:value-of select="$CRLF"/>
</xsl:template> </xsl:template>
<xsl:template match="version" mode="warc"> <xsl:template match="version" name="version" mode="warc">
<xsl:value-of select="$version"/> <xsl:value-of select="$version"/>
<xsl:value-of select="$CRLF"/> <xsl:value-of select="$CRLF"/>
</xsl:template> </xsl:template>
@ -29,7 +29,36 @@
<xsl:value-of select="$CRLF"/> <xsl:value-of select="$CRLF"/>
</xsl:template> </xsl:template>
<xsl:template match="request" mode="warc"> <xsl:template match="record" mode="warc">
<xsl:apply-templates select="header" mode="warc"/>
<xsl:variable name="block">
<xsl:apply-templates select="block" mode="warc"/>
</xsl:variable>
<xsl:variable name="content-length">
<field>
<name>Content-Length</name>
<value>
<xsl:value-of select="string-length($block)"/>
</value>
</field>
</xsl:variable>
<xsl:apply-templates select="$content-length" mode="warc"/>
<xsl:call-template name="CRLF"/>
<xsl:value-of select="$block"/>
<xsl:call-template name="CRLF"/>
<xsl:call-template name="CRLF"/>
</xsl:template>
<xsl:template match="block" mode="warc">
<xsl:apply-templates mode="warc"/>
</xsl:template>
<xsl:template match="header" mode="warc">
<xsl:call-template name="version"/>
<xsl:apply-templates select="*" mode="warc"/>
</xsl:template>
<xsl:template match="request" mode="warc-http">
<line> <line>
<xsl:value-of select="method"/> <xsl:value-of select="method"/>
<xsl:text> </xsl:text> <xsl:text> </xsl:text>
@ -38,20 +67,20 @@
<!-- TODO: get the HTTP version --> <!-- TODO: get the HTTP version -->
<xsl:text>HTTP/1.0</xsl:text> <xsl:text>HTTP/1.0</xsl:text>
</line> </line>
<xsl:apply-templates select="header" mode="warc"/> <xsl:apply-templates select="header" mode="warc-http"/>
</xsl:template> </xsl:template>
<xsl:template match="response" mode="warc"> <xsl:template match="response" mode="warc-http">
<line> <line>
<!-- TODO: get the HTTP version and status--> <!-- TODO: get the HTTP version and status-->
<xsl:text>HTTP/1.1 </xsl:text> <xsl:text>HTTP/1.1 </xsl:text>
<xsl:value-of select="code"/> <xsl:value-of select="code"/>
<xsl:text> OK</xsl:text> <xsl:text> OK</xsl:text>
</line> </line>
<xsl:apply-templates select="header" mode="warc"/> <xsl:apply-templates select="header" mode="warc-http"/>
</xsl:template> </xsl:template>
<xsl:template match="header" mode="warc"> <xsl:template match="header" mode="warc-http">
<field> <field>
<name> <name>
<xsl:value-of select="@name"/> <xsl:value-of select="@name"/>
@ -62,7 +91,7 @@
</field> </field>
</xsl:template> </xsl:template>
<xsl:template match="text()" mode="warc"/> <xsl:template match="text()" mode="warc warc-http"/>
</xsl:stylesheet> </xsl:stylesheet>