Dyomedea
/
owark
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implementing yet another WARC parser (the heritrix one didn't work well with Orbeon due to http client library conflicts).

This commit is contained in:
Eric van der Vlist 2012-04-26 09:48:43 +02:00
parent 307b6d2a72
commit be1a361ab9
11 changed files with 3974 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
archiver/build.xml Normal file
View File

@ -0,0 +1,45 @@
<project name="owark" default="dist" basedir=".">
<description>
Owark build file
</description>
<!-- set global properties for this build -->
<property name="src" location="java/src"/>
<property name="build" location="build"/>
<property name="dist" location="dist"/>
<target name="init">
<!-- Create the time stamp -->
<tstamp/>
<!-- Create the build directory structure used by compile -->
<mkdir dir="${build}"/>
</target>
<target name="compile" depends="init"
description="compile the source " >
<!-- Compile the java code from ${src} into ${build} -->
<javac srcdir="${src}" destdir="${build}">
<classpath>
<pathelement location="java/lib/heritrix-commons-3.1.0.jar"/>
<pathelement location="java/lib/archive-overlay-commons-httpclient-3.1.jar"/>
<pathelement location="/home/vdv/projects/orbeon-forms/build/orbeon-war/WEB-INF/lib/commons-fileupload-1.2.2.jar"/>
<pathelement location="/home/vdv/projects/orbeon-forms/build/orbeon-war/WEB-INF/lib/orbeon.jar"/>
</classpath>
</javac>
</target>
<target name="dist" depends="compile"
description="generate the distribution" >
<!-- Create the distribution directory -->
<mkdir dir="${dist}/lib"/>
<!-- Put everything in ${build} into the MyProject-${DSTAMP}.jar file -->
<jar jarfile="${dist}/lib/owark.jar" basedir="${build}"/>
</target>
<target name="clean"
description="clean up" >
<!-- Delete the ${build} and ${dist} directory trees -->
<delete dir="${build}"/>
<delete dir="${dist}"/>
</target>
</project>

117
archiver/java/src/org/owark/orbeon/FromWarcConverter.java Normal file
View File

@ -0,0 +1,117 @@
/**
* Copyright (C) 2012 Eric van der Vlist.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU Lesser General Public License as published by the Free Software Foundation; either version
* 2.1 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Lesser General Public License for more details.
*
* The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
*/
package org.owark.orbeon;
import org.apache.commons.fileupload.FileItem;
import org.orbeon.oxf.pipeline.api.PipelineContext;
import org.orbeon.oxf.pipeline.api.XMLReceiver;
import org.orbeon.oxf.processor.ProcessorImpl;
import org.orbeon.oxf.processor.ProcessorInputOutputInfo;
import org.orbeon.oxf.processor.ProcessorOutput;
import org.orbeon.oxf.processor.serializer.BinaryTextXMLReceiver;
import org.orbeon.oxf.util.NetUtils;
import org.orbeon.oxf.xml.ContentHandlerHelper;
import org.owark.warc.*;
import java.io.IOException;
/**
* This processor converts a WARC archive into an XML representation
*/
public class FromWarcConverter extends ProcessorImpl {
static public String WARC_ELEMENT_ROOT_NAME = "warc";
static public String RECORD_ELEMENT_NAME = "record";
static public String HEADERS_ELEMENT_NAME = "headers";
static public String HEADER_ELEMENT_NAME = "header";
static public String NAME_ATTRIBUTE_NAME = "name";
static public String CONTENT_ELEMENT_NAME = "content";
public FromWarcConverter() {
addInputInfo(new ProcessorInputOutputInfo(INPUT_DATA));
addOutputInfo(new ProcessorInputOutputInfo(OUTPUT_DATA));
}
@Override
public ProcessorOutput createOutput(String outputName) {
final ProcessorOutput output = new ProcessorOutputImpl(FromWarcConverter.this,outputName) {
@Override
protected void readImpl(PipelineContext pipelineContext, XMLReceiver xmlReceiver) {
// Get FileItem
try {
ContentHandlerHelper helper = new ContentHandlerHelper(xmlReceiver);
helper.startDocument();
helper.startElement(WARC_ELEMENT_ROOT_NAME);
final FileItem fileItem = NetUtils.prepareFileItem(NetUtils.REQUEST_SCOPE);
// Read to OutputStream
readInputAsSAX(pipelineContext, INPUT_DATA, new BinaryTextXMLReceiver(null, fileItem.getOutputStream(), true, false, null, false, false, null, false));
// as an archive
final WarcParser warcParser = new WarcParser(fileItem.getInputStream());
while (warcParser.hasNext()) {
helper.startElement(RECORD_ELEMENT_NAME);
helper.startElement(HEADERS_ELEMENT_NAME);
WarcRecord record = warcParser.next();
WarcRecordHeader recordHeader = record.getHeader();
while (recordHeader.hasNext()) {
WarcField field = recordHeader.next();
helper.startElement(HEADER_ELEMENT_NAME, new String[] {NAME_ATTRIBUTE_NAME, field.getKey()});
helper.text(field.getValue());
helper.endElement();
}
helper.endElement();
helper.startElement(CONTENT_ELEMENT_NAME);
WarcRecordContent content = record.getContent();
if (content.hasRequestLine()) {
helper.startElement("request");
WarcRecordContent.HttpRequestLine request = content.getRequestLine();
helper.element("method", request.getMethod());
helper.element("uri", request.getUri());
helper.element("version", request.getVersion());
helper.endElement();
} else if (content.hasStatusLine()) {
helper.startElement("status");
WarcRecordContent.HttpStatusLine status = content.getStatusLine();
helper.element("version", status.getVersion());
helper.element("status", status.getStatus());
helper.element("reason", status.getReason());
helper.endElement();
}
if (content.hasFields()) {
helper.startElement(HEADERS_ELEMENT_NAME);
while (content.hasNext()) {
WarcField field = content.next();
helper.startElement(HEADER_ELEMENT_NAME, new String[] {NAME_ATTRIBUTE_NAME, field.getKey()});
helper.text(field.getValue());
helper.endElement();
}
helper.endElement();
}
record.skipToEnd();
helper.endElement();
helper.endElement();
}
helper.endElement();
helper.endDocument();
} catch (Exception e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
}
};
addOutput(outputName, output);
return output;
}
}

49
archiver/java/src/org/owark/warc/WarcField.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Copyright (C) 2012 Eric van der Vlist.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU Lesser General Public License as published by the Free Software Foundation; either version
* 2.1 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Lesser General Public License for more details.
*
* The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
*/
package org.owark.warc;
/**
* Created by IntelliJ IDEA.
* User: vdv
* Date: 25 avr. 2012
* Time: 17:56:22
* To change this template use File | Settings | File Templates.
*/
public class WarcField {
private String line;
private String key;
private String value;
public WarcField(String line) {
this.line = line;
int sep = line.indexOf(":");
this.key = line.substring(0, sep).trim();
this.value = line.substring(sep + 1).trim();
}
public String getKey() {
return key;
}
public String getLine() {
return line;
}
public String getValue() {
return value;
}
}

123
archiver/java/src/org/owark/warc/WarcParser.java Normal file
View File

@ -0,0 +1,123 @@
/**
* Copyright (C) 2012 Eric van der Vlist.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU Lesser General Public License as published by the Free Software Foundation; either version
* 2.1 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Lesser General Public License for more details.
*
* The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
*/
package org.owark.warc;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
/**
* Read WARC files
*/
public class WarcParser implements Iterator<WarcRecord> {
public static int BUFFER_SIZE = 1024;
public static String CRLF = "\r\n";
public static String CRLFCRLF = CRLF + CRLF;
public static String MAGIC = "WARC/";
private InputStream is;
private byte[] buffer = new byte[BUFFER_SIZE];
private int index = 0;
private int limit = -1;
private String magic;
private int recordCount;
public WarcParser(InputStream is) {
this.is = is;
resetBuffer();
}
public String getMagic() throws IOException, WarcException {
return this.magic;
}
private void resetBuffer() {
index = 0;
}
private void readUntil(String stringPattern) throws IOException, WarcException {
boolean matches = true;
for (int i=0; i< stringPattern.length() && limit != 0; i++) {
int c = read();
buffer[index ++] = (byte) c;
if (stringPattern.codePointAt(i) != c) {
matches = false;
break;
}
}
if (matches) {
return;
}
readUntil(stringPattern);
}
protected String readLine() throws IOException, WarcException {
readUntil(CRLF);
String line = new String(buffer, 0, index - CRLF.length(), "UTF-8");
resetBuffer();
return line;
}
public boolean hasNext() {
limit = -1;
do {
try {
magic = readLine();
} catch (Exception e) {
return false;
}
} while (! magic.startsWith(MAGIC));
return true;
}
public WarcRecord next() {
recordCount ++;
return new WarcRecord(this);
}
public void remove() {
}
public void setLimit(int limit) {
this.limit = limit;
}
public boolean isLimitReached() {
return limit == 0;
}
public int read() throws IOException {
if (limit == 0) {
return -1;
}
if (limit > 0) {
limit--;
}
int c = is.read();
//System.out.print((char) c);
return c;
}
public int getRecordCount() {
return recordCount;
}
class WarcException extends Exception {}
class BufferOverflowException extends WarcException {}
class BadMagicException extends WarcException {}
}

84
archiver/java/src/org/owark/warc/WarcRecord.java Normal file
View File

@ -0,0 +1,84 @@
/**
* Copyright (C) 2012 Eric van der Vlist.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU Lesser General Public License as published by the Free Software Foundation; either version
* 2.1 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Lesser General Public License for more details.
*
* The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
*/
package org.owark.warc;
import java.io.IOException;
/**
* Created by IntelliJ IDEA.
* User: vdv
* Date: 25 avr. 2012
* Time: 17:29:35
* To change this template use File | Settings | File Templates.
*/
public class WarcRecord {
private WarcParser warcParser;
private WarcRecordHeader header;
private WarcRecordContent content;
public WarcRecord(WarcParser warcParser) {
this.warcParser = warcParser;
}
public Object getMagic() throws IOException, WarcParser.WarcException {
return warcParser.getMagic();
}
public WarcRecordHeader getHeader() {
if (header == null) {
header = new WarcRecordHeader(this);
}
return header;
}
public String readLine() throws IOException, WarcParser.WarcException {
return warcParser.readLine();
}
public String getType() {
return header.getType();
}
public String getContentType() {
return header.getContentType();
}
public WarcRecordContent getContent() {
if (content == null) {
warcParser.setLimit(getContentLength());
content = new WarcRecordContent(this);
}
return content;
}
public int getContentLength() {
return header.getContentLength();
}
public boolean isLimitReached() {
return warcParser.isLimitReached();
}
public int read() throws IOException {
return warcParser.read();
}
public void skipToEnd() throws IOException {
getHeader();
header.skipToEnd();
getContent();
content.skip(getContentLength());
}
}

177
archiver/java/src/org/owark/warc/WarcRecordContent.java Normal file
View File

@ -0,0 +1,177 @@
/**
* Copyright (C) 2012 Eric van der Vlist.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU Lesser General Public License as published by the Free Software Foundation; either version
* 2.1 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Lesser General Public License for more details.
*
* The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
*/
package org.owark.warc;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
/**
* Created by IntelliJ IDEA.
* User: vdv
* Date: 25 avr. 2012
* Time: 19:00:47
* To change this template use File | Settings | File Templates.
*/
public class WarcRecordContent extends InputStream implements Iterator<WarcField> {
private WarcRecord warcRecord;
private Exception e;
private String line;
public WarcRecordContent(WarcRecord warcRecord) {
this.warcRecord = warcRecord;
}
public boolean hasFields() {
return warcRecord.getContentType().equals("application/warc-fields") || isHTTP();
}
public boolean hasNext() {
try {
line = warcRecord.readLine();
} catch (Exception e) {
this.e = e;
}
return ! (warcRecord.isLimitReached() || line.equals(""));
}
public WarcField next() {
if (line == null) {
try {
line = warcRecord.readLine();
} catch (Exception e) {
this.e = e;
}
}
if (line.equals("")) {
line = null;
return null;
}
WarcField field = new WarcField(line);
line = null;
return field;
}
public void remove() {
//To change body of implemented methods use File | Settings | File Templates.
}
@Override
public int read() throws IOException {
return warcRecord.read();
}
public boolean isHTTP() {
return warcRecord.getContentType().startsWith("application/http");
}
public boolean isRequest() {
return warcRecord.getType().equals("request");
}
public HttpStatusLine getStatusLine() throws IOException, WarcParser.WarcException {
return new HttpStatusLine(warcRecord.readLine());
}
public boolean hasStatusLine() {
return isHTTP() && ! isRequest();
}
public boolean hasRequestLine() {
return isHTTP() && isRequest();
}
public Object endOfContent() {
return warcRecord.isLimitReached();
}
public HttpRequestLine getRequestLine() throws IOException, WarcParser.WarcException {
return new HttpRequestLine(warcRecord.readLine());
}
public long getContentLength() {
return warcRecord.getContentLength();
}
public class HttpStatusLine {
private String line;
private String version;
private String status;
private String reason;
public String getLine() {
return line;
}
public String getVersion() {
return version;
}
public String getStatus() {
return status;
}
public String getReason() {
return reason;
}
protected HttpStatusLine(String line) {
this.line = line;
String[] tokens = line.split(" ", 3);
this.version = tokens[0];
this.status = tokens[1];
this.reason = tokens[2];
}
}
public class HttpRequestLine {
private String line;
private String version;
private String method;
private String uri;
public String getLine() {
return line;
}
public String getVersion() {
return version;
}
public String getMethod() {
return method;
}
public String getUri() {
return uri;
}
public HttpRequestLine(String line) {
this.line = line;
String[] tokens = line.split(" ", 3);
this.method = tokens[0];
this.uri = tokens[1];
this.version = tokens[2];
}
}
}

103
archiver/java/src/org/owark/warc/WarcRecordHeader.java Normal file
View File

@ -0,0 +1,103 @@
/**
* Copyright (C) 2012 Eric van der Vlist.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU Lesser General Public License as published by the Free Software Foundation; either version
* 2.1 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Lesser General Public License for more details.
*
* The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
*/
package org.owark.warc;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map;
/**
* Created by IntelliJ IDEA.
* User: vdv
* Date: 25 avr. 2012
* Time: 17:50:01
* To change this template use File | Settings | File Templates.
*/
public class WarcRecordHeader implements Iterator<WarcField> {
public static String WARC_TYPE = "WARC-Type";
public static String CONTENT_TYPE = "Content-Type";
public static String CONTENT_LENGTH = "Content-Length";
private WarcRecord warcRecord;
private String line;
private Exception e;
private Map<String,String> headers;
private boolean endOfHeader = false;
public WarcRecordHeader(WarcRecord warcRecord) {
this.warcRecord = warcRecord;
headers = new Hashtable<String, String>();
}
public boolean hasNext() {
if (endOfHeader) {
return false;
}
if (line == null) {
try {
line = warcRecord.readLine();
} catch (Exception e) {
this.e = e;
return false;
}
}
if (line.equals("")) {
endOfHeader = true;
return false;
}
return true;
}
public WarcField next() {
if (endOfHeader) {
return null;
}
if (line == null) {
try {
line = warcRecord.readLine();
} catch (Exception e) {
this.e = e;
return null;
}
}
WarcField item = new WarcField(line);
line = null;
headers.put(item.getKey(), item.getValue());
return item;
}
public String getType() {
return headers.get(WARC_TYPE);
}
public void remove() {
}
public String getContentType() {
return headers.get(CONTENT_TYPE);
}
public int getContentLength() {
return Integer.parseInt(headers.get(CONTENT_LENGTH));
}
public void skipToEnd() {
while (hasNext()) {
next();
}
}
}

297
archiver/java/test/org/owark/warc/WarcParserTest.java Normal file
View File

@ -0,0 +1,297 @@
/**
* Copyright (C) 2012 Eric van der Vlist.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU Lesser General Public License as published by the Free Software Foundation; either version
* 2.1 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Lesser General Public License for more details.
*
* The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
*/
package org.owark.warc;
import org.junit.Assert;
import org.junit.Test;
import java.io.*;
/**
* Test cases for WarcParser
*/
public class WarcParserTest {
private static WarcParser warcParser;
@Test
public void testDyomedea() throws IOException, WarcParser.WarcException {
// WARC
File file = new File("/home/vdv/projects/owark/archiver/java/test/org/owark/warc/dyomedea.warc");
WarcParser warcParser = new WarcParser(new FileInputStream(file));
Assert.assertEquals(true, warcParser.hasNext());
// RECORD
WarcRecord record = warcParser.next();
Assert.assertEquals("WARC/1.0", warcParser.getMagic());
Assert.assertNotNull(record);
Assert.assertEquals("WARC/1.0", record.getMagic());
// HEADER
WarcRecordHeader header = record.getHeader();
Assert.assertNotNull(header);
Assert.assertNull(header.getType());
Assert.assertEquals(true, header.hasNext());
WarcField headerItem = header.next();
Assert.assertNotNull(headerItem);
Assert.assertEquals(WarcRecordHeader.WARC_TYPE, headerItem.getKey());
Assert.assertEquals("warcinfo", headerItem.getValue());
Assert.assertEquals("warcinfo", header.getType());
Assert.assertEquals("warcinfo", record.getType());
Assert.assertEquals(true, header.hasNext());
headerItem = header.next();
Assert.assertNotNull(headerItem);
Assert.assertEquals("WARC-Date", headerItem.getKey());
Assert.assertEquals("2012-04-23T10:05:24Z", headerItem.getValue());
headerItem = header.next();
headerItem = header.next();
headerItem = header.next();
headerItem = header.next();
Assert.assertNotNull(headerItem);
Assert.assertEquals("Content-Length", headerItem.getKey());
Assert.assertEquals("369", headerItem.getValue());
Assert.assertEquals(false, header.hasNext());
headerItem = header.next();
Assert.assertNull(headerItem);
Assert.assertEquals("application/warc-fields", record.getContentType());
Assert.assertEquals(369, record.getContentLength());
// Content
WarcRecordContent content = record.getContent();
Assert.assertNotNull(content);
Assert.assertEquals(true, content.hasFields());
Assert.assertEquals(false, content.isHTTP());
Assert.assertEquals(false, content.hasStatusLine());
Assert.assertEquals(false, content.hasRequestLine());
Assert.assertEquals(true, content.hasNext());
WarcField field = content.next();
Assert.assertEquals(false, content.endOfContent());
Assert.assertNotNull(field);
Assert.assertEquals("software", field.getKey());
Assert.assertEquals("Heritrix/3.1.0 http://crawler.archive.org", field.getValue());
field = content.next();
field = content.next();
field = content.next();
field = content.next();
field = content.next();
field = content.next();
field = content.next();
field = content.next();
Assert.assertNotNull(field);
Assert.assertEquals("http-header-user-agent", field.getKey());
Assert.assertEquals("Mozilla/5.0 (compatible; heritrix/3.1.0 +http://owark.org)", field.getValue());
Assert.assertEquals(false, content.hasNext());
Assert.assertEquals(true, content.endOfContent());
// Next record
Assert.assertEquals(true, warcParser.hasNext());
record = warcParser.next();
Assert.assertNotNull(record);
// Header
header = record.getHeader();
Assert.assertNotNull(header);
Assert.assertNull(header.getType());
Assert.assertEquals(true, header.hasNext());
headerItem = header.next();
Assert.assertNotNull(headerItem);
Assert.assertEquals(WarcRecordHeader.WARC_TYPE, headerItem.getKey());
Assert.assertEquals("response", headerItem.getValue());
header.skipToEnd();
// Content
content = record.getContent();
Assert.assertNotNull(content);
Assert.assertEquals(false, content.hasFields());
Assert.assertEquals(false, content.isHTTP());
Assert.assertEquals(false, content.hasStatusLine());
Assert.assertEquals(false, content.hasRequestLine());
Assert.assertEquals(false, content.endOfContent());
BufferedReader reader = new BufferedReader(new InputStreamReader(content, "UTF-8"));
String line = reader.readLine();
Assert.assertEquals("20120423100524", line);
line = reader.readLine();
Assert.assertEquals("dyomedea.com.\t\t1800\tIN\tA\t95.142.167.137", line);
line = reader.readLine();
Assert.assertEquals(true, content.endOfContent());
Assert.assertNull(line);
// Next record
Assert.assertEquals(true, warcParser.hasNext());
record = warcParser.next();
Assert.assertNotNull(record);
// Header
header = record.getHeader();
Assert.assertNotNull(header);
Assert.assertNull(header.getType());
Assert.assertEquals(true, header.hasNext());
headerItem = header.next();
Assert.assertNotNull(headerItem);
Assert.assertEquals(WarcRecordHeader.WARC_TYPE, headerItem.getKey());
Assert.assertEquals("response", headerItem.getValue());
header.skipToEnd();
// Content
content = record.getContent();
Assert.assertNotNull(content);
Assert.assertEquals(true, content.hasFields());
Assert.assertEquals(true, content.isHTTP());
Assert.assertEquals(false, content.isRequest());
Assert.assertEquals(true, content.hasStatusLine());
Assert.assertEquals(false, content.hasRequestLine());
WarcRecordContent.HttpStatusLine status = content.getStatusLine();
Assert.assertNotNull(status);
Assert.assertEquals("HTTP/1.1 404 Introuvable", status.getLine());
Assert.assertEquals("HTTP/1.1", status.getVersion());
Assert.assertEquals("404", status.getStatus());
Assert.assertEquals("Introuvable", status.getReason());
field = content.next();
Assert.assertNotNull(field);
Assert.assertEquals("Date", field.getKey());
Assert.assertEquals("Mon, 23 Apr 2012 10:05:27 GMT", field.getValue());
field = content.next();
field = content.next();
field = content.next();
field = content.next();
field = content.next();
field = content.next();
Assert.assertNotNull(field);
Assert.assertEquals("Connection", field.getKey());
Assert.assertEquals("close", field.getValue());
Assert.assertEquals(false, content.hasNext());
Assert.assertEquals(false, content.endOfContent());
reader = new BufferedReader(new InputStreamReader(content, "UTF-8"));
line = reader.readLine();
Assert.assertEquals("<html><head><title>Apache Tomcat/6.0.24 - Rapport d'erreur</title>", line.substring(0, line.indexOf("<style>")));
line = reader.readLine();
Assert.assertNull(line);
Assert.assertEquals(true, content.endOfContent());
// Next record
Assert.assertEquals(true, warcParser.hasNext());
record = warcParser.next();
Assert.assertNotNull(record);
// Header
header = record.getHeader();
Assert.assertNotNull(header);
Assert.assertNull(header.getType());
Assert.assertEquals(true, header.hasNext());
headerItem = header.next();
Assert.assertNotNull(headerItem);
Assert.assertEquals(WarcRecordHeader.WARC_TYPE, headerItem.getKey());
Assert.assertEquals("request", headerItem.getValue());
header.skipToEnd();
// Content
content = record.getContent();
Assert.assertNotNull(content);
Assert.assertEquals(true, content.hasFields());
Assert.assertEquals(true, content.isHTTP());
Assert.assertEquals(true, content.isRequest());
Assert.assertEquals(false, content.hasStatusLine());
Assert.assertEquals(true, content.hasRequestLine());
WarcRecordContent.HttpRequestLine request = content.getRequestLine();
Assert.assertEquals("GET /robots.txt HTTP/1.0", request.getLine());
Assert.assertEquals("GET", request.getMethod());
Assert.assertEquals("/robots.txt", request.getUri());
Assert.assertEquals("HTTP/1.0", request.getVersion());
field = content.next();
Assert.assertNotNull(field);
Assert.assertEquals("User-Agent", field.getKey());
Assert.assertEquals("Mozilla/5.0 (compatible; heritrix/3.1.0 +http://owark.org)", field.getValue());
field = content.next();
field = content.next();
field = content.next();
Assert.assertNotNull(field);
Assert.assertEquals("Host", field.getKey());
Assert.assertEquals("dyomedea.com", field.getValue());
Assert.assertEquals(false, content.hasNext());
Assert.assertEquals(true, content.endOfContent());
// Skip record
Assert.assertEquals(true, warcParser.hasNext());
record = warcParser.next();
Assert.assertNotNull(record);
record.skipToEnd();
Assert.assertEquals(true, warcParser.hasNext());
record = warcParser.next();
// Header
header = record.getHeader();
Assert.assertNotNull(header);
Assert.assertNull(header.getType());
Assert.assertEquals(true, header.hasNext());
headerItem = header.next();
Assert.assertNotNull(headerItem);
Assert.assertEquals(WarcRecordHeader.WARC_TYPE, headerItem.getKey());
Assert.assertEquals("response", headerItem.getValue());
record.skipToEnd();
// Go to last record
while (warcParser.hasNext()) {
record = warcParser.next();
Assert.assertNotNull(record);
record.skipToEnd();
}
Assert.assertEquals(69, warcParser.getRecordCount());
Assert.assertEquals("metadata", record.getType());
}
@Test
public void skipToEnd() throws IOException, WarcParser.WarcException {
File file = new File("/home/vdv/projects/owark/archiver/java/test/org/owark/warc/dyomedea.warc");
WarcParser warcParser = new WarcParser(new FileInputStream(file));
Assert.assertEquals(true, warcParser.hasNext());
WarcRecord record = warcParser.next();
WarcRecordHeader header = record.getHeader();
while (header.hasNext()) {
Assert.assertNotNull(header.next());
}
WarcRecordContent content = record.getContent();
while (content.hasNext()) {
Assert.assertNotNull(content.next());
}
record.skipToEnd();
}
} ;

2963
archiver/java/test/org/owark/warc/dyomedea.warc Normal file
View File

File diff suppressed because it is too large Load Diff

11
archiver/pipelines/actions/package-heritrix-warc.xpl
View File

@ -7,7 +7,7 @@
<p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xforms="http://www.w3.org/2002/xforms" <p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xforms="http://www.w3.org/2002/xforms"
xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:saxon="http://saxon.sf.net/" xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:saxon="http://saxon.sf.net/"
xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary"> xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary" xmlns:owk="http://owark.org/orbeon/processors">
<p:param name="data" type="input"/> <p:param name="data" type="input"/>
@ -33,6 +33,15 @@
</p:input> </p:input>
<p:output name="data" id="warc"/> <p:output name="data" id="warc"/>
</p:processor> </p:processor>
<p:processor name="owk:from-warc-converter">
<p:input name="data" href="#warc"/>
<p:output name="data" id="warc-xml" debug="warc-xml"/>
</p:processor>
<p:processor name="oxf:null-serializer">
<p:input name="data" href="#warc-xml"/>
</p:processor>
<!-- Store it in a temp file --> <!-- Store it in a temp file -->
<p:processor name="oxf:file-serializer"> <p:processor name="oxf:file-serializer">

6
archiver/pipelines/config/processors-local.xml Normal file
View File

@ -0,0 +1,6 @@
<processors xmlns:owk="http://owark.org/orbeon/processors">
<processor name="owk:from-warc-converter">
<class name="org.owark.orbeon.FromWarcConverter"/>
</processor>
</processors>