public class HDFSOutputConnector extends BaseOutputConnector
Modifier and Type | Class and Description |
---|---|
protected static class |
HDFSOutputConnector.CheckConnectionThread |
protected static class |
HDFSOutputConnector.CreateFileThread |
protected static class |
HDFSOutputConnector.DeleteFileThread |
protected static class |
HDFSOutputConnector.GetSessionThread |
Modifier and Type | Field and Description |
---|---|
static String |
_rcsid |
protected static String[] |
activitiesList |
static String |
INGEST_ACTIVITY
Ingestion activity
|
protected long |
lastSessionFetch |
protected String |
nameNodeHost |
protected String |
nameNodePort |
protected String |
nameNodeProtocol |
static String |
REMOVE_ACTIVITY
Document removal activity
|
protected HDFSSession |
session |
protected static long |
timeToRelease |
protected String |
user |
currentContext, params
DOCUMENTSTATUS_ACCEPTED, DOCUMENTSTATUS_REJECTED
Constructor and Description |
---|
HDFSOutputConnector()
Constructor.
|
Modifier and Type | Method and Description |
---|---|
int |
addOrReplaceDocumentWithException(String documentURI,
VersionContext pipelineDescription,
RepositoryDocument document,
String authorityNameString,
IOutputAddActivity activities)
Add (or replace) a document in the output data store using the connector.
|
String |
check()
Test the connection.
|
protected void |
checkConnection() |
protected void |
closeSession() |
void |
connect(ConfigParams configParams)
Connect.
|
protected void |
createFile(org.apache.hadoop.fs.Path path,
InputStream input,
IOutputAddActivity activities,
String documentURI) |
protected void |
deleteFile(org.apache.hadoop.fs.Path path,
IOutputRemoveActivity activities,
String documentURI) |
void |
disconnect()
Close the connection.
|
String[] |
getActivitiesList()
Return the list of activities that this connector supports (i.e.
|
String |
getFormCheckJavascriptMethodName(int connectionSequenceNumber)
Obtain the name of the form check javascript method to call.
|
String |
getFormPresaveCheckJavascriptMethodName(int connectionSequenceNumber)
Obtain the name of the form presave check javascript method to call.
|
VersionContext |
getPipelineDescription(Specification spec)
Get an output version string, given an output specification.
|
protected HDFSSession |
getSession()
Set up a session
|
protected static void |
handleIOException(IOException e)
Handle IOException
|
protected static void |
handleJSONException(org.json.JSONException e)
Handle JSONException
|
protected static void |
handleURISyntaxException(URISyntaxException e)
Handle URISyntaxException
|
boolean |
isConnected()
This method is called to assess whether to count this connector instance should
actually be counted as being connected.
|
void |
outputConfigurationBody(IThreadContext threadContext,
IHTTPOutput out,
Locale locale,
ConfigParams parameters,
String tabName)
Output the configuration body section.
|
void |
outputConfigurationHeader(IThreadContext threadContext,
IHTTPOutput out,
Locale locale,
ConfigParams parameters,
List<String> tabsArray)
Output the configuration header section.
|
void |
outputSpecificationBody(IHTTPOutput out,
Locale locale,
Specification os,
int connectionSequenceNumber,
int actualSequenceNumber,
String tabName)
Output the specification body section.
|
void |
outputSpecificationHeader(IHTTPOutput out,
Locale locale,
Specification os,
int connectionSequenceNumber,
List<String> tabsArray)
Output the specification header section.
|
void |
poll() |
String |
processConfigurationPost(IThreadContext threadContext,
IPostParameters variableContext,
Locale locale,
ConfigParams parameters)
Process a configuration post.
|
String |
processSpecificationPost(IPostParameters variableContext,
Locale locale,
Specification os,
int connectionSequenceNumber)
Process a specification post.
|
void |
removeDocument(String documentURI,
String outputDescription,
IOutputRemoveActivity activities)
Remove a document using the connector.
|
void |
viewConfiguration(IThreadContext threadContext,
IHTTPOutput out,
Locale locale,
ConfigParams parameters)
View configuration.
|
void |
viewSpecification(IHTTPOutput out,
Locale locale,
Specification os,
int connectionSequenceNumber)
View specification.
|
addOrReplaceDocument, checkDateIndexable, checkDocumentIndexable, checkDocumentIndexable, checkDocumentIndexable, checkLengthIndexable, checkLengthIndexable, checkMimeTypeIndexable, checkMimeTypeIndexable, checkMimeTypeIndexable, checkURLIndexable, checkURLIndexable, getOutputDescription, noteAllRecordsRemoved, noteJobComplete, outputSpecificationBody, outputSpecificationBody, outputSpecificationHeader, outputSpecificationHeader, outputSpecificationHeader, processSpecificationPost, processSpecificationPost, requestInfo, viewSpecification, viewSpecification
clearThreadContext, deinstall, getConfiguration, install, outputConfigurationBody, outputConfigurationHeader, outputConfigurationHeader, pack, packFixedList, packList, packList, processConfigurationPost, setThreadContext, unpack, unpackFixedList, unpackList, viewConfiguration
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
clearThreadContext, deinstall, getConfiguration, install, setThreadContext
public static final String _rcsid
public static final String INGEST_ACTIVITY
public static final String REMOVE_ACTIVITY
protected static final String[] activitiesList
protected String nameNodeProtocol
protected String nameNodeHost
protected String nameNodePort
protected String user
protected HDFSSession session
protected long lastSessionFetch
protected static final long timeToRelease
public String[] getActivitiesList()
getActivitiesList
in interface IOutputConnector
getActivitiesList
in class BaseOutputConnector
public void connect(ConfigParams configParams)
connect
in interface IConnector
connect
in class BaseConnector
configParameters
- is the set of configuration parameters, which
in this case describe the target appliance, basic auth configuration, etc. (This formerly came
out of the ini file.)public boolean isConnected()
isConnected
in interface IConnector
isConnected
in class BaseConnector
public void disconnect() throws ManifoldCFException
disconnect
in interface IConnector
disconnect
in class BaseConnector
ManifoldCFException
public void poll() throws ManifoldCFException
poll
in interface IConnector
poll
in class BaseConnector
ManifoldCFException
protected void closeSession() throws ManifoldCFException
ManifoldCFException
protected HDFSSession getSession() throws ManifoldCFException, ServiceInterruption
public String check() throws ManifoldCFException
check
in interface IConnector
check
in class BaseConnector
ManifoldCFException
public VersionContext getPipelineDescription(Specification spec) throws ManifoldCFException, ServiceInterruption
getPipelineDescription
in interface IPipelineConnector
getPipelineDescription
in class BaseOutputConnector
spec
- is the current output specification for the job that is doing the crawling.ManifoldCFException
ServiceInterruption
public int addOrReplaceDocumentWithException(String documentURI, VersionContext pipelineDescription, RepositoryDocument document, String authorityNameString, IOutputAddActivity activities) throws ManifoldCFException, ServiceInterruption, IOException
addOrReplaceDocumentWithException
in interface IPipelineConnector
addOrReplaceDocumentWithException
in class BaseOutputConnector
documentURI
- is the URI of the document. The URI is presumed to be the unique identifier which the output data store will use to process
and serve the document. This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.pipelineDescription
- includes the description string that was constructed for this document by the getOutputDescription() method.document
- is the document data to be processed (handed to the output data store).authorityNameString
- is the name of the authority responsible for authorizing any access tokens passed in with the repository document. May be null.activities
- is the handle to an object that the implementer of a pipeline connector may use to perform operations, such as logging processing activity,
or sending a modified document to the next stage in the pipeline.IOException
- only if there's a stream error reading the document data.ManifoldCFException
ServiceInterruption
public void removeDocument(String documentURI, String outputDescription, IOutputRemoveActivity activities) throws ManifoldCFException, ServiceInterruption
removeDocument
in interface IOutputConnector
removeDocument
in class BaseOutputConnector
documentURI
- is the URI of the document. The URI is presumed to be the unique identifier which the output data store will use to process
and serve the document. This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.outputDescription
- is the last description string that was constructed for this document by the getOutputDescription() method above.activities
- is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.ManifoldCFException
ServiceInterruption
public void outputConfigurationHeader(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, List<String> tabsArray) throws ManifoldCFException, IOException
outputConfigurationHeader
in interface IConnector
outputConfigurationHeader
in class BaseConnector
threadContext
- is the local thread context.out
- is the output to which any HTML should be sent.parameters
- are the configuration parameters, as they currently exist, for this connection being configured.tabsArray
- is an array of tab names. Add to this array any tab names that are specific to the connector.ManifoldCFException
IOException
public void outputConfigurationBody(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, String tabName) throws ManifoldCFException, IOException
outputConfigurationBody
in interface IConnector
outputConfigurationBody
in class BaseConnector
threadContext
- is the local thread context.out
- is the output to which any HTML should be sent.parameters
- are the configuration parameters, as they currently exist, for this connection being configured.tabName
- is the current tab name.ManifoldCFException
IOException
public String processConfigurationPost(IThreadContext threadContext, IPostParameters variableContext, Locale locale, ConfigParams parameters) throws ManifoldCFException
processConfigurationPost
in interface IConnector
processConfigurationPost
in class BaseConnector
threadContext
- is the local thread context.variableContext
- is the set of variables available from the post, including binary file post information.parameters
- are the configuration parameters, as they currently exist, for this connection being configured.ManifoldCFException
public void viewConfiguration(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters) throws ManifoldCFException, IOException
viewConfiguration
in interface IConnector
viewConfiguration
in class BaseConnector
threadContext
- is the local thread context.out
- is the output to which any HTML should be sent.parameters
- are the configuration parameters, as they currently exist, for this connection being configured.ManifoldCFException
IOException
public String getFormCheckJavascriptMethodName(int connectionSequenceNumber)
getFormCheckJavascriptMethodName
in interface IPipelineConnector
getFormCheckJavascriptMethodName
in class BaseOutputConnector
connectionSequenceNumber
- is the unique number of this connection within the job.public String getFormPresaveCheckJavascriptMethodName(int connectionSequenceNumber)
getFormPresaveCheckJavascriptMethodName
in interface IPipelineConnector
getFormPresaveCheckJavascriptMethodName
in class BaseOutputConnector
connectionSequenceNumber
- is the unique number of this connection within the job.public void outputSpecificationHeader(IHTTPOutput out, Locale locale, Specification os, int connectionSequenceNumber, List<String> tabsArray) throws ManifoldCFException, IOException
outputSpecificationHeader
in interface IPipelineConnector
outputSpecificationHeader
in class BaseOutputConnector
out
- is the output to which any HTML should be sent.locale
- is the preferred local of the output.os
- is the current pipeline specification for this connection.connectionSequenceNumber
- is the unique number of this connection within the job.tabsArray
- is an array of tab names. Add to this array any tab names that are specific to the connector.ManifoldCFException
IOException
public void outputSpecificationBody(IHTTPOutput out, Locale locale, Specification os, int connectionSequenceNumber, int actualSequenceNumber, String tabName) throws ManifoldCFException, IOException
outputSpecificationBody
in interface IPipelineConnector
outputSpecificationBody
in class BaseOutputConnector
out
- is the output to which any HTML should be sent.locale
- is the preferred local of the output.os
- is the current pipeline specification for this job.connectionSequenceNumber
- is the unique number of this connection within the job.actualSequenceNumber
- is the connection within the job that has currently been selected.tabName
- is the current tab name.ManifoldCFException
IOException
public String processSpecificationPost(IPostParameters variableContext, Locale locale, Specification os, int connectionSequenceNumber) throws ManifoldCFException
processSpecificationPost
in interface IPipelineConnector
processSpecificationPost
in class BaseOutputConnector
variableContext
- contains the post data, including binary file-upload information.locale
- is the preferred local of the output.os
- is the current pipeline specification for this job.connectionSequenceNumber
- is the unique number of this connection within the job.ManifoldCFException
public void viewSpecification(IHTTPOutput out, Locale locale, Specification os, int connectionSequenceNumber) throws ManifoldCFException, IOException
viewSpecification
in interface IPipelineConnector
viewSpecification
in class BaseOutputConnector
out
- is the output to which any HTML should be sent.locale
- is the preferred local of the output.connectionSequenceNumber
- is the unique number of this connection within the job.os
- is the current pipeline specification for this job.ManifoldCFException
IOException
protected static void handleURISyntaxException(URISyntaxException e) throws ManifoldCFException, ServiceInterruption
protected static void handleJSONException(org.json.JSONException e) throws ManifoldCFException, ServiceInterruption
protected static void handleIOException(IOException e) throws ManifoldCFException, ServiceInterruption
protected void createFile(org.apache.hadoop.fs.Path path, InputStream input, IOutputAddActivity activities, String documentURI) throws ManifoldCFException, ServiceInterruption
protected void deleteFile(org.apache.hadoop.fs.Path path, IOutputRemoveActivity activities, String documentURI) throws ManifoldCFException, ServiceInterruption
protected void checkConnection() throws ManifoldCFException, ServiceInterruption