message: Update python code styling

diff:
@@ -77,6 +77,7 @@ tf.app.flags.DEFINE_integer('num_top_predictions', 5,
DATA_URL = 'http://download.tensorflow.org/models/inception_v4_2016_09_09.tar.gz'
# pylint: enable=line-too-long
+
def create_readable_names_for_imagenet_labels():
"""Create a dict mapping label id to human readable string.
@@ -104,11 +105,13 @@ def create_readable_names_for_imagenet_labels():
dest_directory = FLAGS.model_dir
- synset_list = [s.strip() for s in open(os.path.join(dest_directory, 'imagenet_lsvrc_2015_synsets.txt')).readlines()]
+ synset_list = [s.strip() for s in open(os.path.join(
+ dest_directory, 'imagenet_lsvrc_2015_synsets.txt')).readlines()]
num_synsets_in_ilsvrc = len(synset_list)
assert num_synsets_in_ilsvrc == 1000
- synset_to_human_list = open(os.path.join(dest_directory, 'imagenet_metadata.txt')).readlines()
+ synset_to_human_list = open(os.path.join(
+ dest_directory, 'imagenet_metadata.txt')).readlines()
num_synsets_in_all_imagenet = len(synset_to_human_list)
assert num_synsets_in_all_imagenet == 21842
@@ -129,6 +132,7 @@ def create_readable_names_for_imagenet_labels():
return labels_to_names
+
def run_inference_on_image(image):
"""Runs inference on an image.
@@ -148,12 +152,15 @@ def run_inference_on_image(image):
with tf.Graph().as_default():
image = tf.image.decode_jpeg(image_string, channels=3)
- processed_image = inception_preprocessing.preprocess_image(image, image_size, image_size, is_training=False)
+ processed_image = inception_preprocessing.preprocess_image(
+ image, image_size, image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
- # Create the model, use the default arg scope to configure the batch norm parameters.
+ # Create the model, use the default arg scope to configure the batch
+ # norm parameters.
with slim.arg_scope(inception.inception_v4_arg_scope()):
- logits, _ = inception.inception_v4(processed_images, num_classes=1001, is_training=False)
+ logits, _ = inception.inception_v4(
+ processed_images, num_classes=1001, is_training=False)
probabilities = tf.nn.softmax(logits)
init_fn = slim.assign_from_checkpoint_fn(
@@ -164,7 +171,8 @@ def run_inference_on_image(image):
init_fn(sess)
_, probabilities = sess.run([image, probabilities])
probabilities = probabilities[0, 0:]
- sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x:x[1])]
+ sorted_inds = [i[0] for i in sorted(
+ enumerate(-probabilities), key=lambda x:x[1])]
names = create_readable_names_for_imagenet_labels()
top_k = FLAGS.num_top_predictions
@@ -172,6 +180,7 @@ def run_inference_on_image(image):
index = sorted_inds[i]
print('%s (score = %.5f)' % (names[index], probabilities[index]))
+
def util_download(url, dest_directory):
"""Downloads the file.
@@ -193,6 +202,7 @@ def util_download(url, dest_directory):
statinfo = os.stat(filepath)
print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
+
def util_download_tar(url, dest_directory):
"""Downloads a file and extracts it.
@@ -215,6 +225,7 @@ def util_download_tar(url, dest_directory):
print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
tarfile.open(filepath, 'r:gz').extractall(dest_directory)
+
def maybe_download_and_extract():
"""Download and extract model tar file."""
dest_directory = FLAGS.model_dir
@@ -224,13 +235,14 @@ def maybe_download_and_extract():
util_download_tar(DATA_URL, dest_directory)
# pylint: disable=line-too-long
if not tf.gfile.Exists(os.path.join(dest_directory, 'imagenet_lsvrc_2015_synsets.txt')):
- util_download('https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_lsvrc_2015_synsets.txt', dest_directory)
+ util_download(
+ 'https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_lsvrc_2015_synsets.txt', dest_directory)
if not tf.gfile.Exists(os.path.join(dest_directory, 'imagenet_metadata.txt')):
- util_download('https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_metadata.txt', dest_directory)
+ util_download(
+ 'https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_metadata.txt', dest_directory)
# pylint: enable=line-too-long
-
def main(_):
maybe_download_and_extract()
image = FLAGS.image_file

diff:
@@ -49,11 +49,11 @@ slim = tf.contrib.slim
import requests
import json
-json.encoder.FLOAT_REPR = lambda o: format(o, '.2f') # JSON serialization of floats
+json.encoder.FLOAT_REPR = lambda o: format(
+ o, '.2f') # JSON serialization of floats
from time import time
-
import flask
@@ -71,12 +71,14 @@ tf.app.flags.DEFINE_string(
"""imagenet_lsvrc_2015_synsets.txt, and """
"""imagenet_metadata.txt.""")
tf.app.flags.DEFINE_integer('port', '8764', """Server PORT, default:8764""")
-tf.app.flags.DEFINE_string('log', 'inception.log', """Log file name, default: inception.log""")
+tf.app.flags.DEFINE_string('log', 'inception.log',
+ """Log file name, default: inception.log""")
# pylint: disable=line-too-long
DATA_URL = 'http://download.tensorflow.org/models/inception_v4_2016_09_09.tar.gz'
# pylint: enable=line-too-long
+
def create_readable_names_for_imagenet_labels():
"""Create a dict mapping label id to human readable string.
@@ -104,11 +106,13 @@ def create_readable_names_for_imagenet_labels():
dest_directory = FLAGS.model_dir
- synset_list = [s.strip() for s in open(os.path.join(dest_directory, 'imagenet_lsvrc_2015_synsets.txt')).readlines()]
+ synset_list = [s.strip() for s in open(os.path.join(
+ dest_directory, 'imagenet_lsvrc_2015_synsets.txt')).readlines()]
num_synsets_in_ilsvrc = len(synset_list)
assert num_synsets_in_ilsvrc == 1000
- synset_to_human_list = open(os.path.join(dest_directory, 'imagenet_metadata.txt')).readlines()
+ synset_to_human_list = open(os.path.join(
+ dest_directory, 'imagenet_metadata.txt')).readlines()
num_synsets_in_all_imagenet = len(synset_to_human_list)
assert num_synsets_in_all_imagenet == 21842
@@ -129,6 +133,7 @@ def create_readable_names_for_imagenet_labels():
return labels_to_names
+
def util_download(url, dest_directory):
"""Downloads the file.
@@ -150,6 +155,7 @@ def util_download(url, dest_directory):
statinfo = os.stat(filepath)
print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
+
def util_download_tar(url, dest_directory):
"""Downloads a file and extracts it.
@@ -172,6 +178,7 @@ def util_download_tar(url, dest_directory):
print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
tarfile.open(filepath, 'r:gz').extractall(dest_directory)
+
def maybe_download_and_extract():
"""Download and extract model tar file."""
dest_directory = FLAGS.model_dir
@@ -181,29 +188,36 @@ def maybe_download_and_extract():
util_download_tar(DATA_URL, dest_directory)
# pylint: disable=line-too-long
if not tf.gfile.Exists(os.path.join(dest_directory, 'imagenet_lsvrc_2015_synsets.txt')):
- util_download('https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_lsvrc_2015_synsets.txt', dest_directory)
+ util_download(
+ 'https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_lsvrc_2015_synsets.txt', dest_directory)
if not tf.gfile.Exists(os.path.join(dest_directory, 'imagenet_metadata.txt')):
- util_download('https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_metadata.txt', dest_directory)
+ util_download(
+ 'https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_metadata.txt', dest_directory)
# pylint: enable=line-too-long
+
def current_time():
"""
Returns current time in milli seconds
"""
return int(1000 * time())
+
class Classifier(flask.Flask):
'''
Classifier Service class
'''
+
def __init__(self, name):
super(Classifier, self).__init__(name)
maybe_download_and_extract()
import logging
from logging.handlers import RotatingFileHandler
- file_handler = RotatingFileHandler(FLAGS.log, maxBytes=1024 * 1024 * 100, backupCount=20)
+ file_handler = RotatingFileHandler(
+ FLAGS.log, maxBytes=1024 * 1024 * 100, backupCount=20)
file_handler.setLevel(logging.INFO)
- formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+ formatter = logging.Formatter(
+ "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setFormatter(formatter)
self.logger.addHandler(file_handler)
self.names = imagenet.create_readable_names_for_imagenet_labels()
@@ -212,15 +226,17 @@ class Classifier(flask.Flask):
def classify(self, image_string, topk):
dest_directory = FLAGS.model_dir
-
with tf.Graph().as_default():
image = tf.image.decode_jpeg(image_string, channels=3)
- processed_image = inception_preprocessing.preprocess_image(image, self.image_size, self.image_size, is_training=False)
+ processed_image = inception_preprocessing.preprocess_image(
+ image, self.image_size, self.image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
- # Create the model, use the default arg scope to configure the batch norm parameters.
+ # Create the model, use the default arg scope to configure the
+ # batch norm parameters.
with slim.arg_scope(inception.inception_v4_arg_scope()):
- logits, _ = inception.inception_v4(processed_images, num_classes=1001, is_training=False)
+ logits, _ = inception.inception_v4(
+ processed_images, num_classes=1001, is_training=False)
probabilities = tf.nn.softmax(logits)
init_fn = slim.assign_from_checkpoint_fn(
@@ -231,8 +247,8 @@ class Classifier(flask.Flask):
init_fn(sess)
_, probabilities = sess.run([image, probabilities])
probabilities = probabilities[0, 0:]
- sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x:x[1])]
-
+ sorted_inds = [i[0] for i in sorted(
+ enumerate(-probabilities), key=lambda x:x[1])]
res = []
for i in range(topk):
@@ -245,6 +261,7 @@ class Classifier(flask.Flask):
from flask import Flask, request, abort, g, Response, jsonify
app = Classifier(__name__)
+
def get_remotefile(url, success=200, timeout=10):
"""
Given HTTP URL, this api gets the content of it
@@ -260,6 +277,7 @@ def get_remotefile(url, success=200, timeout=10):
pass
return None, None
+
@app.route("/")
def index():
"""
@@ -303,6 +321,7 @@ def ping_pong():
"""
return "pong"
+
@app.route("/inception/v4/classify", methods=["GET", "POST"])
def classify_image():
"""
@@ -343,6 +362,7 @@ def classify_image():
res['classnames'] = classnames
return Response(response=json.dumps(res), status=200, mimetype="application/json")
+
def main(_):
if not app.debug:
print("Serving on port %d" % FLAGS.port)

original_message: Update python code styling


message: Updated dockerfile to launch the service

change_type: MODIFY

diff:
@@ -46,4 +46,4 @@ ENV LD_PRELOAD="/usr/lib/libtcmalloc_minimal.so.4"
# expose API port, this is the default port
EXPOSE 8764
-#CMD inceptionapi.py
+CMD inceptionapi.py

license: Apache License 2.0

repo: apache/tika


message: Updated model repo link to official tensorflow's

diff:
@@ -20,7 +20,7 @@ MAINTAINER Thamme Gowda <thammegowda@apache.org> and Kranthi Kiran GV <kkranthi@
# Install missing part of ubuntu core + python stuff
RUN apt-get update && \
- apt-get install -y python-pip python-dev wget libtcmalloc-minimal4 git
+ apt-get install -y python-pip python-dev wget libtcmalloc-minimal4 git unzip
# Install tensorflow and other dependencies
RUN \
@@ -28,8 +28,9 @@ RUN \
pip install flask requests
# Get the TF-slim dependencies
-# Forked for future compatibility
-RUN git clone https://github.com/KranthiGV/models
+# Downloading from a specific commit for future compatibility
+RUN wget https://github.com/tensorflow/models/archive/c15fada28113eca32dc98d6e3bec4755d0d5b4c2.zip && \
+ unzip c15fada28113eca32dc98d6e3bec4755d0d5b4c2.zip
# TODO: Change the URL to Apache/Tika Repo when this PR gets merged
RUN \
@@ -39,7 +40,7 @@ RUN \
# clean up cache, so we can publish smaller image to hub
RUN apt-get clean
-ENV PYTHONPATH="$PYTHONPATH:/models/slim"
+ENV PYTHONPATH="$PYTHONPATH:/models-c15fada28113eca32dc98d6e3bec4755d0d5b4c2/slim"
ENV LD_PRELOAD="/usr/lib/libtcmalloc_minimal.so.4"


message: Reduced disk I/O

diff:
@@ -37,12 +37,10 @@ RUN \
wget https://raw.githubusercontent.com/KranthiGV/tika/TIKA-2306/tika-parsers/src/main/resources/org/apache/tika/parser/recognition/tf/inceptionapi.py -O /usr/bin/inceptionapi.py && \
chmod +x /usr/bin/inceptionapi.py
-
# clean up cache, so we can publish smaller image to hub
RUN apt-get clean
ENV PYTHONPATH="$PYTHONPATH:/models-c15fada28113eca32dc98d6e3bec4755d0d5b4c2/slim"
-ENV LD_PRELOAD="/usr/lib/libtcmalloc_minimal.so.4"
-
+#ENV LD_PRELOAD="/usr/lib/libtcmalloc_minimal.so.4"
# expose API port, this is the default port
EXPOSE 8764

diff:
@@ -223,37 +223,37 @@ class Classifier(flask.Flask):
self.names = imagenet.create_readable_names_for_imagenet_labels()
self.image_size = inception.inception_v4.default_image_size
- def classify(self, image_string, topk):
- dest_directory = FLAGS.model_dir
-
- with tf.Graph().as_default():
- image = tf.image.decode_jpeg(image_string, channels=3)
+ self.image_str_placeholder = tf.placeholder(tf.string)
+ image = tf.image.decode_jpeg(self.image_str_placeholder, channels=3)
processed_image = inception_preprocessing.preprocess_image(
image, self.image_size, self.image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
-
# Create the model, use the default arg scope to configure the
# batch norm parameters.
with slim.arg_scope(inception.inception_v4_arg_scope()):
logits, _ = inception.inception_v4(
processed_images, num_classes=1001, is_training=False)
- probabilities = tf.nn.softmax(logits)
+ self.probabilities = tf.nn.softmax(logits)
+ dest_directory = FLAGS.model_dir
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(dest_directory, 'inception_v4.ckpt'),
slim.get_model_variables('InceptionV4'))
- with tf.Session() as sess:
- init_fn(sess)
- _, probabilities = sess.run([image, probabilities])
- probabilities = probabilities[0, 0:]
+ self.sess = tf.Session()
+ init_fn(self.sess)
+
+ def classify(self, image_string, topk):
+ eval_probabilities = self.sess.run(self.probabilities, feed_dict={
+ self.image_str_placeholder: image_string})
+ eval_probabilities = eval_probabilities[0, 0:]
sorted_inds = [i[0] for i in sorted(
- enumerate(-probabilities), key=lambda x:x[1])]
+ enumerate(-eval_probabilities), key=lambda x:x[1])]
res = []
for i in range(topk):
index = sorted_inds[i]
- score = float(probabilities[index])
+ score = float(eval_probabilities[index])
res.append((index, self.names[index], score))
return res

original_message: Reduced disk I/O


message: Remove redundancy. Not updating classify_image.py since it has no effect on runtime performance

diff:
@@ -40,7 +40,7 @@ RUN \
# clean up cache, so we can publish smaller image to hub
RUN apt-get clean
ENV PYTHONPATH="$PYTHONPATH:/models-c15fada28113eca32dc98d6e3bec4755d0d5b4c2/slim"
-#ENV LD_PRELOAD="/usr/lib/libtcmalloc_minimal.so.4"
+ENV LD_PRELOAD="/usr/lib/libtcmalloc_minimal.so.4"
# expose API port, this is the default port
EXPOSE 8764

diff:
@@ -169,7 +169,7 @@ def run_inference_on_image(image):
with tf.Session() as sess:
init_fn(sess)
- _, probabilities = sess.run([image, probabilities])
+ probabilities = sess.run(probabilities)
probabilities = probabilities[0, 0:]
sorted_inds = [i[0] for i in sorted(
enumerate(-probabilities), key=lambda x:x[1])]


message: change scope of jai-imageio-core

diff:
<artifactId>jempbox</artifactId>
<version>${jempbox.version}</version>
</dependency>
+ <!-- TIKA-2338: PDFBox declares the jai-imageio-core dependency
+ as optional, but we prefer to have it always to support
+ tiff and other image formats -->
+ <dependency>
+ <groupId>com.github.jai-imageio</groupId>
+ <artifactId>jai-imageio-core</artifactId>
+ <version>1.3.1</version>
+ </dependency>
<!-- TIKA-370: PDFBox declares the Bouncy Castle dependencies
as optional, but we prefer to have them always to avoid
problems with encrypted PDFs. -->
<scope>test</scope>
</dependency>
<!-- Copied from PDFBox:
- For legal reasons (incompatible license), jai-imageio-core is to be used
+ For legal reasons (incompatible license), jai-imageio-jpeg2000 is to be used
only in the tests and may not be distributed. See also LEGAL-195 -->
- <dependency>
- <groupId>com.github.jai-imageio</groupId>
- <artifactId>jai-imageio-core</artifactId>
- <version>1.3.1</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>com.github.jai-imageio</groupId>
<artifactId>jai-imageio-jpeg2000</artifactId>

original_message: change scope of jai-imageio-core (TIKA-2338)


message: fix detection of emails inside mbox

diff:
Release 1.17 - ???
+ * Fix detection of emails extracted from mbox (TIKA-2456)
+
* Add OverrideDetector and allow PSTParser to specify body content type
as text or html -- to avoid incorrect auto-detection of
rfc/mbox, etc. (TIKA-2454)

diff:
@@ -103,6 +103,7 @@ public class MboxParser extends AbstractParser {
Queue<String> multiline = new LinkedList<String>();
mailMetadata.add(EMAIL_FROMLINE_METADATA, curLine.substring(MBOX_RECORD_DIVIDER.length()));
mailMetadata.set(Metadata.CONTENT_TYPE, "message/rfc822");
+ mailMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE, "message/rfc822");
curLine = reader.readLine();
if (curLine == null) {
break;

diff:
@@ -152,4 +152,19 @@ public class MboxParserTest {
assertContains("When a Mapper completes", handler.toString());
}
+
+ @Test
+ public void testOverrideDetector() throws Exception {
+ ContentHandler handler = new BodyContentHandler();
+ Metadata metadata = new Metadata();
+ ParseContext context = new ParseContext();
+ context.set(Parser.class, new AutoDetectParser());
+
+ try (InputStream stream = getStream("/test-documents/single_mail.mbox")) {
+ mboxParser.parse(stream, handler, metadata, context);
+ }
+
+ Metadata firstMail = mboxParser.getTrackingMetadata().get(0);
+ assertEquals("message/rfc822", firstMail.get(Metadata.CONTENT_TYPE));
+ }
}

original_message: TIKA-2456: fix detection of emails inside mbox


message: load custom mimetypes XML from sys prop

diff:
Release 1.17 - ???
+ * Load external custom mimetypes XML from system property
+ tika.custom-mimetypes (TIKA-2460).
+
* Extract number of tiffs in a multi-page tiff (TIKA-2451).
* Fix detection of emails extracted from mbox (TIKA-2456).

diff:
*/
package org.apache.tika.mime;
+import java.io.File;
import java.io.InputStream;
import java.io.IOException;
import java.net.URL;
@@ -30,6 +31,12 @@ import org.w3c.dom.Document;
*/
public class MimeTypesFactory {
+ /**
+ * System property to set a path to an additional external custom mimetypes
+ * XML file to be loaded.
+ */
+ public static final String CUSTOM_MIMES_SYS_PROP = "tika.custom-mimetypes";
+
/**
* Creates an empty instance; same as calling new MimeTypes().
*
@@ -139,6 +146,8 @@ public class MimeTypesFactory {
* override mimetypes found will loaded afterwards.
* The file paths will be interpreted by the specified class
* loader in getResource().
+ * It will also load custom mimetypes from the system property
+ * {@link #CUSTOM_MIMES_SYS_PROP}, if specified.
*
* @param coreFilePath The main MimeTypes file to load
* @param extensionFilePath The name of extension MimeType files to load afterwards
@@ -167,6 +176,16 @@ public class MimeTypesFactory {
urls.add(coreURL);
urls.addAll(extensionURLs);
+ String customMimesPath = System.getProperty(CUSTOM_MIMES_SYS_PROP);
+ if(customMimesPath != null){
+ File externalFile = new File(customMimesPath);
+ if(!externalFile.exists())
+ throw new IOException(
+ "Specified custom mimetypes file not found: " + customMimesPath);
+ URL externalURL = externalFile.toURI().toURL();
+ urls.add(externalURL);
+ }
+
return create( urls.toArray(new URL[urls.size()]) );
}
}

diff:
@@ -233,6 +233,22 @@ public class MimeTypesReaderTest {
}
}
+ private class CustomClassLoader extends ClassLoader{
+ }
+
+ /**
+ * TIKA-2460 Test loading of custom-mimetypes.xml from sys prop.
+ */
+ @Test
+ public void testExternalMimeTypes() throws Exception {
+ System.setProperty(MimeTypesFactory.CUSTOM_MIMES_SYS_PROP,
+ "src/test/resources/org/apache/tika/mime/external-mimetypes.xml");
+ MimeTypes mimeTypes = MimeTypes.getDefaultMimeTypes(new CustomClassLoader());
+ Metadata m = new Metadata();
+ m.add(Metadata.RESOURCE_NAME_KEY, "test.external.mime.type");
+ assertEquals("external/mime-type", mimeTypes.detect(null, m).toString());
+ }
+
@Test
public void testGetExtensionForPowerPoint() throws Exception {
MimeType mt = this.mimeTypes.forName("application/vnd.ms-powerpoint");

change_type: ADD

diff:
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<mime-info>
+ <mime-type type="external/mime-type">
+ <glob pattern="*.external.mime.type" />
+ </mime-type>
+</mime-info>

original_message: TIKA-2460: load custom mimetypes XML from sys prop


message: support for tif in pdfs

diff:
@@ -5,6 +5,8 @@ Release 2.0.0 - ???
Other changes
+ * Support for TIFF images in PDF files (TIKA-2338)
+
* Detection of full encrypted 7z files (TIKA-2568)
* Various new mimes and typo fixes in tika-mimetypes.xml

diff:
<version>1.6.5</version>
<scope>test</scope>
</dependency>
- <!-- Copied from PDFBox:
- For legal reasons (incompatible license), jai-imageio-core is to be used
- only in the tests and may not be distributed. See also LEGAL-195-->
+ <!-- jai-imageio-core is allowed since LEGAL-304 -->
<dependency>
<groupId>com.github.jai-imageio</groupId>
<artifactId>jai-imageio-core</artifactId>
<version>1.3.1</version>
- <scope>test</scope>
</dependency>
+ <!-- For legal reasons (incompatible license), jai-imageio-jpeg2000 is to be used
+ only in the tests and may not be distributed. See also LEGAL-195 -->
<dependency>
<groupId>com.github.jai-imageio</groupId>
<artifactId>jai-imageio-jpeg2000</artifactId>

original_message: TIKA-2338 support for tif in pdfs


message: great optimization in ForkParser

diff:
@@ -13,7 +13,11 @@ Release 2.0.0 - ???
Other changes
-Release 1.25 - 11/23/2020
+Release 1.26 - ???
+
+ * Great optimization in ForkParser (TIKA-3237).
+
+Release 1.25 - 11/25/2020
* Fix inconsistent license in xmpcore (TIKA-3204).

diff:
@@ -67,7 +67,7 @@ class ContentHandlerProxy implements ContentHandler, ForkProxy {
try {
if (string != null) {
output.writeBoolean(true);
- output.writeUTF(string);
+ writeString(string);
} else {
output.writeBoolean(false);
}
@@ -76,13 +76,26 @@ class ContentHandlerProxy implements ContentHandler, ForkProxy {
}
}
+ /**
+ * Breaks the string in 21,845 size chunks to not throw UTFDataFormatException at least in Oracle JDK 8.
+ */
+ private void writeString(String string) throws IOException {
+ int max = 65535 / 3;
+ int frags = (int) Math.ceil((double) string.length() / max);
+ output.writeInt(frags);
+ int i = 0;
+ while (i < frags) {
+ int end = (i < frags - 1) ? (i + 1) * max : string.length();
+ output.writeUTF(string.substring(i * max, end));
+ i++;
+ }
+ }
+
private void sendCharacters(char[] ch, int start, int length)
throws SAXException {
try {
- output.writeInt(length);
- for (int i = 0; i < length; i++) {
- output.writeChar(ch[start + i]);
- }
+ writeString(new String(ch, start, length));
+
} catch (IOException e) {
throw new SAXException("Unexpected fork proxy problem", e);
}

diff:
@@ -89,19 +89,23 @@ class ContentHandlerResource implements ForkResource {
private String readString(DataInputStream input) throws IOException {
if (input.readBoolean()) {
- return input.readUTF();
+ return readStringUTF(input);
} else {
return null;
}
}
private char[] readCharacters(DataInputStream input) throws IOException {
- int n = input.readInt();
- char[] ch = new char[n];
- for (int i = 0; i < n; i++) {
- ch[i] = input.readChar();
+ return readStringUTF(input).toCharArray();
+ }
+
+ private String readStringUTF(DataInputStream input) throws IOException {
+ int frags = input.readInt();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < frags; i++) {
+ sb.append(input.readUTF());
}
- return ch;
+ return sb.toString();
}
}

diff:
@@ -21,6 +21,8 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -413,6 +415,18 @@ public class ForkParserTest extends TikaTest {
assertEquals("/embed1.xml", m1.get(RecursiveParserWrapperHandler.EMBEDDED_RESOURCE_PATH));
}
+ @Test
+ public void testNoUTFDataFormatException() throws Exception {
+ ContentHandlerProxy proxy = new ContentHandlerProxy(0);
+ DataOutputStream output = new DataOutputStream(new ByteArrayOutputStream());
+ proxy.init(null, output);
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 65536; i++) {
+ sb.append(1);
+ }
+ proxy.skippedEntity(sb.toString());
+ }
+
//use this to test that the wrapper handler is acted upon by the server but not proxied back
private static class ToFileHandler extends AbstractRecursiveParserWrapperHandler {

original_message: TIKA-3237: great optimization in ForkParser


diff:
@@ -17,6 +17,8 @@ Release 1.26 - ???
* Great optimization in ForkParser (TIKA-3237).
+ * Fix parsing of emails attached to other emails in PST files (TIKA-3004).
+
Release 1.25 - 11/25/2020
* Fix inconsistent license in xmpcore (TIKA-3204).

diff:
@@ -25,12 +25,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Set;
-import com.pff.PSTAttachment;
-import com.pff.PSTException;
-import com.pff.PSTFile;
-import com.pff.PSTFolder;
-import com.pff.PSTMessage;
-import com.pff.PSTRecipient;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
@@ -48,6 +42,13 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
+import com.pff.PSTAttachment;
+import com.pff.PSTException;
+import com.pff.PSTFile;
+import com.pff.PSTFolder;
+import com.pff.PSTMessage;
+import com.pff.PSTRecipient;
+
/**
* Parser for MS Outlook PST email storage files
*/
@@ -115,6 +116,23 @@ public class OutlookPSTParser extends AbstractParser {
if (pstFolder.getContentCount() > 0) {
PSTMessage pstMail = (PSTMessage) pstFolder.getNextChild();
while (pstMail != null) {
+ parseMailAndAttachments(handler, pstMail, embeddedExtractor);
+ pstMail = (PSTMessage) pstFolder.getNextChild();
+ }
+ }
+
+ if (pstFolder.hasSubfolders()) {
+ for (PSTFolder pstSubFolder : pstFolder.getSubFolders()) {
+ handler.startElement("div", createAttribute("class", "email-folder"));
+ handler.element("h1", pstSubFolder.getDisplayName());
+ parseFolder(handler, pstSubFolder, embeddedExtractor);
+ handler.endElement("div");
+ }
+ }
+ }
+
+ private void parseMailAndAttachments(XHTMLContentHandler handler, PSTMessage pstMail,
+ EmbeddedDocumentExtractor embeddedExtractor) throws SAXException, IOException, TikaException {
AttributesImpl attributes = new AttributesImpl();
attributes.addAttribute("", "class", "class", "CDATA", "embedded");
attributes.addAttribute("", "id", "id", "CDATA", pstMail.getInternetMessageId());
@@ -131,19 +149,6 @@ public class OutlookPSTParser extends AbstractParser {
parserMailItem(handler, pstMail, mailMetadata, embeddedExtractor);
handler.endElement("div");
-
- pstMail = (PSTMessage) pstFolder.getNextChild();
- }
- }
-
- if (pstFolder.hasSubfolders()) {
- for (PSTFolder pstSubFolder : pstFolder.getSubFolders()) {
- handler.startElement("div", createAttribute("class", "email-folder"));
- handler.element("h1", pstSubFolder.getDisplayName());
- parseFolder(handler, pstSubFolder, embeddedExtractor);
- handler.endElement("div");
- }
- }
}
private void parserMailItem(XHTMLContentHandler handler, PSTMessage pstMail, Metadata mailMetadata,
@@ -231,6 +236,12 @@ public class OutlookPSTParser extends AbstractParser {
try {
PSTAttachment attach = email.getAttachment(i);
+ PSTMessage attachedEmail = attach.getEmbeddedPSTMessage();
+ if (attachedEmail != null) {
+ parseMailAndAttachments(xhtml, attachedEmail, embeddedExtractor);
+ continue;
+ }
+
// Get the filename; both long and short filenames can be used for attachments
String filename = attach.getLongFilename();
if (filename.isEmpty()) {

original_message: TIKA-3004: Fix parsing of emails attached to other emails in PST files


original_message: TIKA-3004: update test to search for attached mail body


message: Created basic structure

diff:
<module>tika-translate</module>
<module>tika-example</module>
<module>tika-java7</module>
+ <module>tika-transcribe</module>
</modules>
<profiles>

diff:
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.transcribe;
+
+import org.apache.tika.exception.TikaException;
+
+import java.io.IOException;
+
+
+/**
+ * Interface for Transcriber services.
+ *
+ * @since Tika TODO
+ */
+public interface Transcriber {
+ /**
+ * @return
+ * @throws TikaException When there is an error translating.
+ * @throws java.io.IOException
+ * @since TODO
+ */
+ public String transcribe() throws TikaException, IOException;
+}

diff:
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parent</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../tika-parent/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>tika-transcribe</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Tika transcribe</name>
+ <url>http://tika.apache.org/</url>
+
+<dependencies>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+</dependencies>
+</project>
\ No newline at end of file

diff:
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.transcribe.transcribe;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.transcribe.Transcriber;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+public class AmazonTranscribe implements Transcriber {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AmazonTranscribe.class);
+
+ boolean available; // Flag for whether or not translation is available.
+ String clientId, clientSecret; // Keys used for the API calls.
+
+ public AmazonTranscribe() {
+ Properties props = new Properties();
+ InputStream stream;
+ //TODO
+ }
+
+ @Override
+ public String transcribe() throws TikaException, IOException {
+ //TODO
+ return null;
+ }
+}

diff:
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.tika.language.translate.amazontranscribe

diff:
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+AWS_ACCESS_KEY=key
+AWS_SECRET_KEY=key

diff:
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.transcibe.transcibe;
+
+import org.apache.tika.transcribe.transcribe.AmazonTranscribe;
+import org.junit.Before;
+
+public class AmazonTranscribeTest {
+ AmazonTranscribe transcribe;
+
+ @Before
+ public void setUp() {
+ transcribe = new AmazonTranscribe();
+ }
+ //TODO
+}

original_message: Created basic structure


message: pom build fix

diff:
<artifactId>junit</artifactId>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <version>${maven.bundle.version}</version>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Bundle-DocURL>${project.url}</Bundle-DocURL>
+ <Bundle-Activator>
+ org.apache.tika.parser.internal.Activator
+ </Bundle-Activator>
+ <Import-Package>
+ org.w3c.dom,
+ org.apache.tika.*,
+ *;resolution:=optional
+ </Import-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${rat.version}</version>
+ <configuration>
+ <excludes>
+ <exclude>src/main/java/org/apache/tika/parser/txt/Charset*.java</exclude>
+ <exclude>src/test/resources/test-documents/**</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifestEntries>
+ <Automatic-Module-Name>org.apache.tika.translate</Automatic-Module-Name>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+
+ <pluginManagement>
+ <plugins>
+ <!-- This plugin's configuration is used to store Eclipse m2e -->
+ <!-- settings only. It has no influence on the Maven build itself. -->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ <version>${maven.scr.version}</version>
+ <goals>
+ <goal>scr</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute />
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
</project>
\ No newline at end of file

original_message: pom build fix


message: added amazonaws dependency

diff:
<packaging>bundle</packaging>
<name>Apache Tika transcribe</name>
<url>http://tika.apache.org/</url>
-
+ <!--TODO use latest aws version or the one defined in the tika-parent-->
<dependencies>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-transcribe</artifactId>
+ <version>${aws.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>${aws.version}</version>
+ </dependency>
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>

original_message: added amazonaws dependency


message:
> removed aws from core
> refactored Transcriber interface

diff:
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-transcribe</artifactId>
- <version>1.11.937</version>
- <scope>compile</scope>
- </dependency>
</dependencies>
<build>

diff:
@@ -21,70 +21,57 @@ import org.apache.tika.exception.TikaException;
import java.io.IOException;
-import com.amazonaws.services.transcribe.model.LanguageCode;
-
/**
* Interface for Transcriber services.
- *
* @since Tika TODO
*/
public interface Transcriber {
/**
+ * Transcribe the given audio file.
+ * @param filePath
* @return
- * @param fileName
- * @param jobName
* @throws TikaException When there is an error translating.
- * @throws java.io.IOException
+ * @throws IOException
* @since TODO
*/
- public void startTranscribeAudio(String fileName, String jobName) throws TikaException, IOException;
+ public void startTranscribeAudio(String filePath) throws TikaException, IOException;
/**
- * @return
- * @param fileName
+ * Transcribe the given the audio file and the source language.
+ * @param filePath
* @param sourceLanguage
- * @param jobName
+ * @return
* @throws TikaException When there is an error translating.
* @throws java.io.IOException
* @since TODO
*/
- public void startTranscribeAudio(String fileName, LanguageCode sourceLanguage, String jobName) throws TikaException, IOException;
+ public void startTranscribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException;
/**
+ * Transcribe the given the video file.
+ * @param filePath
* @return
- * @param fileName
- * @param jobName
* @throws TikaException When there is an error translating.
* @throws java.io.IOException
* @since TODO
*/
- public void startTranscribeVideo(String fileName, String jobName) throws TikaException, IOException;
+ public void startTranscribeVideo(String filePath) throws TikaException, IOException;
/**
- * @return
- * @param fileName
- * @param jobName
+ * Transcribe the given the video file and the source language.
+ * @param filePath
* @param sourceLanguage
+ * @return
* @throws TikaException When there is an error translating.
* @throws java.io.IOException
* @since TODO
*/
- public void startTranscribeVideo(String fileName, LanguageCode sourceLanguage, String jobName) throws TikaException, IOException;
+ public void startTranscribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException;
/**
- * Gets transcription result from S3
- * @param key
- * @return
- */
- public String getTranscriptResult(String key);
-
- /**
- * Upload file to s3
- * @param bucketName
- * @param fileName
- * @param filePath
+ * @return true if this Transcriber is probably able to translate right now.
+ * @since Tika TODO
*/
- public void uploadFileToBucket(String bucketName, String fileName, String filePath);
-
+ public boolean isAvailable();
}


message: Package Name refactoring, more generic interface, changes in implementation

diff:
@@ -24,21 +24,24 @@ import java.io.IOException;
/**
* Interface for Transcriber services.
+ *
* @since Tika TODO
*/
public interface Transcriber {
/**
* Transcribe the given audio file.
+ *
* @param filePath
* @return
* @throws TikaException When there is an error translating.
* @throws IOException
* @since TODO
*/
- public void startTranscribeAudio(String filePath) throws TikaException, IOException;
+ public String startTranscribeAudio(String filePath) throws TikaException, IOException;
/**
* Transcribe the given the audio file and the source language.
+ *
* @param filePath
* @param sourceLanguage
* @return
@@ -46,20 +49,22 @@ public interface Transcriber {
* @throws java.io.IOException
* @since TODO
*/
- public void startTranscribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException;
+ public String startTranscribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException;
/**
* Transcribe the given the video file.
+ *
* @param filePath
* @return
* @throws TikaException When there is an error translating.
* @throws java.io.IOException
* @since TODO
*/
- public void startTranscribeVideo(String filePath) throws TikaException, IOException;
+ public String startTranscribeVideo(String filePath) throws TikaException, IOException;
/**
* Transcribe the given the video file and the source language.
+ *
* @param filePath
* @param sourceLanguage
* @return
@@ -67,7 +72,7 @@ public interface Transcriber {
* @throws java.io.IOException
* @since TODO
*/
- public void startTranscribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException;
+ public String startTranscribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException;
/**
* @return true if this Transcriber is probably able to translate right now.

change_type: RENAME

diff:
* limitations under the License.
*/
-package org.apache.tika.transcribe.transcribe;
-import java.io.File;
+package org.apache.tika.transcribe;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.transcribe.AmazonTranscribeAsync;
import com.amazonaws.services.transcribe.model.*;
import org.apache.tika.exception.TikaException;
-import org.apache.tika.transcribe.Transcriber;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.transcribe.AmazonTranscribeAsync;
-
+import java.io.File;
import java.io.IOException;
import java.util.Properties;
public class AmazonTranscribe implements Transcriber {
- private AmazonTranscribeAsync amazonTranscribe;
-
- private AmazonS3 amazonS3;
+ public static final String PROPERTIES_FILE = "transcribe.amazon.properties";
+ public static final String ID_PROPERTY = "transcribe.AWS_ACCESS_KEY";
+ public static final String SECRET_PROPERTY = "transcribe.AWS_SECRET_KEY";
+ public static final String DEFAULT_ID = "dummy-id";
+ public static final String DEFAULT_SECRET = "dummy-secret";
+ public static final String DEFAULT_BUCKET = "dummy-bucket";
+ public static final String BUCKET_NAME = "transcribe.BUCKET_NAME";
private static final Logger LOG = LoggerFactory.getLogger(AmazonTranscribe.class);
-
+ private AmazonTranscribeAsync amazonTranscribe;
+ private AmazonS3 amazonS3;
private String bucketName;
-
private boolean isAvailable; // Flag for whether or not translation is available.
-
+ private String jobName;
private String clientId;
-
private String clientSecret; // Keys used for the API calls.
-
// private HashSet<String> validSourceLanguages = new HashSet<>(Arrays.asList("en-US", "en-GB", "es-US", "fr-CA", "fr-FR", "en-AU",
// "it-IT", "de-DE", "pt-BR", "ja-JP", "ko-KR")); // Valid inputs to StartStreamTranscription for language of source file (audio)
- public AmazonTranscribe() {
- this.isAvailable = true;
+ public AmazonTranscribe(String jobName) {
+ this.jobName = jobName;
Properties config = new Properties();
try {
config.load(AmazonTranscribe.class
.getResourceAsStream(
- "transcribe.amazon.properties"));
- this.clientId = config.getProperty("transcribe.AWS_ACCESS_KEY");
- this.clientSecret = config.getProperty("transcribe.AWS_SECRET_KEY");
- this.bucketName = config.getProperty("transcribe.BUCKET_NAME");
+ PROPERTIES_FILE));
+ this.clientId = config.getProperty(ID_PROPERTY);
+ this.clientSecret = config.getProperty(SECRET_PROPERTY);
+ this.bucketName = config.getProperty(BUCKET_NAME);
+ this.isAvailable = checkAvailable();
} catch (Exception e) {
LOG.warn("Exception reading config file", e);
@@ -68,126 +69,139 @@ public class AmazonTranscribe implements Transcriber {
}
}
+ /**
+ * Call this method in order to upload a file to the Amazon S3 bucket.
+ *
+ * @param bucketName
+ * @param filePath
+ * @param fullfilePath
+ */
+ public void uploadFileToBucket(String bucketName, String filePath, String fullfilePath) {
+ PutObjectRequest request = new PutObjectRequest(bucketName, filePath, new File(fullfilePath));
+ amazonS3.putObject(request);
+ }
/**
- * Audio to text function without language specification
- * @param fileName
- * @return Transcribed text
+ * Gets Transcription result from AWS S3 bucket given bucketName and key
+ *
+ * @param key
+ * @return
+ */
+ public String getTranscriptResult(String key) {
+ TranscriptionJob transcriptionJob = retrieveObjectWhenJobCompleted(key);
+ if (transcriptionJob != null && !TranscriptionJobStatus.FAILED.equals(transcriptionJob.getTranscriptionJobStatus())) {
+ return amazonS3.getObjectAsString(this.bucketName, key + ".json");
+ } else
+ return null;
+ }
+
+ /**
+ * Private helper function to get object from s3
+ *
+ * @param key
+ * @return
+ */
+ private TranscriptionJob retrieveObjectWhenJobCompleted(String key) {
+ GetTranscriptionJobRequest getTranscriptionJobRequest = new GetTranscriptionJobRequest();
+ getTranscriptionJobRequest.setTranscriptionJobName(key);
+ while (true) {
+ GetTranscriptionJobResult innerResult = amazonTranscribe.getTranscriptionJob(getTranscriptionJobRequest);
+ String status = innerResult.getTranscriptionJob().getTranscriptionJobStatus();
+ if (TranscriptionJobStatus.COMPLETED.name().equals(status) ||
+ TranscriptionJobStatus.FAILED.name().equals(status)) {
+ return innerResult.getTranscriptionJob();
+ }
+ }
+ }
+
+ /**
+ * @param filePath
* @throws TikaException
* @throws IOException
*/
@Override
- public void startTranscribeAudio(String fileName, String jobName) throws TikaException, IOException {
- if (!isAvailable())
- return;
+ public String startTranscribeAudio(String filePath) throws TikaException, IOException {
+ if (!isAvailable) return "";
StartTranscriptionJobRequest startTranscriptionJobRequest = new StartTranscriptionJobRequest();
Media media = new Media();
- media.setMediaFileUri(amazonS3.getUrl(bucketName, fileName).toString());
+ media.setMediaFileUri(amazonS3.getUrl(bucketName, filePath).toString());
startTranscriptionJobRequest.withMedia(media)
+// .withLanguageCode(LanguageCode.EnUS)
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
+ return getTranscriptResult(jobName + ".json");
}
- /**
- * Audio to text function with language specification
- * @param fileName
- * @param sourceLanguage
- * @return Transcribed text
- * @throws TikaException
- * @throws IOException
- */
@Override
- public void startTranscribeAudio(String fileName, LanguageCode sourceLanguage, String jobName) throws TikaException, IOException {
- if (!isAvailable())
- return;
+ public String startTranscribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException {
+ if (!isAvailable) return "";
StartTranscriptionJobRequest startTranscriptionJobRequest = new StartTranscriptionJobRequest();
Media media = new Media();
- media.setMediaFileUri(amazonS3.getUrl(bucketName, fileName).toString());
+ media.setMediaFileUri(amazonS3.getUrl(bucketName, filePath).toString());
startTranscriptionJobRequest.withMedia(media)
- .withLanguageCode(sourceLanguage)
+ .withLanguageCode(LanguageCode.valueOf(sourceLanguage))
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
+ return getTranscriptResult(jobName + ".json");
}
@Override
- public void startTranscribeVideo(String fileName, String jobName) throws TikaException, IOException {
- if (!isAvailable())
- return;
+ public String startTranscribeVideo(String filePath) throws TikaException, IOException {
+ if (!isAvailable) return "";
//TODO
-
+ return "";
}
- /**
- * Audio to text function with language specification
- * @param fileName
- * @param sourceLanguage
- * @return Transcribed text
- * @throws TikaException
- * @throws IOException
- */
@Override
- public void startTranscribeVideo(String fileName, LanguageCode sourceLanguage, String jobName) throws TikaException, IOException {
- if (!isAvailable())
- return;
- //boolean validSourceLanguageFlag = validSourceLanguages.contains(sourceLanguage); // Checks if sourceLanguage in validSourceLanguages O(1) lookup time
-
- //if (!validSourceLanguageFlag) { // Throws TikaException if the input sourceLanguage is not present in validSourceLanguages
- // throw new TikaException("Provided Source Language is Not Valid. Run without language parameter or please select one of: " +
- // "en-US, en-GB, es-US, fr-CA, fr-FR, en-AU, it-IT, de-DE, pt-BR, ja-JP, ko-KR"); }
+ public String startTranscribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException {
+ if (!isAvailable) return "";
//TODO
-
+ return "";
}
- /**
- * @return Valid AWS Credentials
- */
+ @Override
public boolean isAvailable() {
- return this.isAvailable;
+ return isAvailable;
}
- /** Gets Transcriptioni result from AWS S3 bucket given bucketNamee and key
- * @param key
- * @return
+ /**
+ * Sets the client Id for the translator API.
+ *
+ * @param id The ID to set.
*/
- @Override
- public String getTranscriptResult(String key) {
- TranscriptionJob transcriptionJob = retrieveObjectWhenJobCompleted(key);
- if (transcriptionJob != null && !TranscriptionJobStatus.FAILED.equals(transcriptionJob.getTranscriptionJobStatus())) {
- return amazonS3.getObjectAsString(this.bucketName, key + ".json");
- } else
- return null;
+ public void setId(String id) {
+ this.clientId = id;
+ this.isAvailable = checkAvailable();
}
/**
- * Private helper function to get object from s3
- * @param key
- * @return
+ * Sets the client secret for the translator API.
+ *
+ * @param secret The secret to set.
*/
- private TranscriptionJob retrieveObjectWhenJobCompleted(String key) {
- GetTranscriptionJobRequest getTranscriptionJobRequest = new GetTranscriptionJobRequest();
- getTranscriptionJobRequest.setTranscriptionJobName(key);
-
- while (true) {
- GetTranscriptionJobResult innerResult = amazonTranscribe.getTranscriptionJob(getTranscriptionJobRequest);
- String status = innerResult.getTranscriptionJob().getTranscriptionJobStatus();
- if (TranscriptionJobStatus.COMPLETED.name().equals(status) ||
- TranscriptionJobStatus.FAILED.name().equals(status)) {
- return innerResult.getTranscriptionJob();
- }
- }
+ public void setSecret(String secret) {
+ this.clientSecret = secret;
+ this.isAvailable = checkAvailable();
}
/**
- * Call this method in order to upload a file to the Amazon S3 bucket.
- * @param bucketName
- * @param fileName
- * @param fullFileName
+ * Sets the client secret for the translator API.
+ *
+ * @param bucket The bucket to set.
*/
- @Override
- public void uploadFileToBucket(String bucketName, String fileName, String fullFileName) {
- PutObjectRequest request = new PutObjectRequest(bucketName, fileName, new File(fullFileName));
- amazonS3.putObject(request);
+ public void setBucket(String bucket) {
+ this.bucketName = bucket;
+ this.isAvailable = checkAvailable();
+ }
+
+ private boolean checkAvailable() {
+ return clientId != null &&
+ !clientId.equals(DEFAULT_ID) &&
+ clientSecret != null &&
+ !clientSecret.equals(DEFAULT_SECRET) &&
+ bucketName != null &&
+ !bucketName.equals(DEFAULT_BUCKET);
}
}

diff:
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.transcibe.transcibe;
+package org.apache.tika.transcibe;
-import org.apache.tika.transcribe.transcribe.AmazonTranscribe;
+import org.apache.tika.transcribe.AmazonTranscribe;
import org.junit.Before;
import org.junit.Test;

diff:
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.transcibe.transcibe;
+package org.apache.tika.transcibe;
-import org.apache.tika.transcribe.transcribe.AmazonTranscribe;
+import org.apache.tika.transcribe.AmazonTranscribe;
import org.junit.Before;
public class AmazonTranscribeTest {


message:
uploadFileToBucket is now a private method.
key -> jobName
TODO add documentation for the methods

diff:
@@ -70,50 +70,21 @@ public class AmazonTranscribe implements Transcriber {
}
/**
- * Call this method in order to upload a file to the Amazon S3 bucket.
+ * Constructs a new
+ * {@link PutObjectRequest} object to upload a file to the
+ * specified bucket and jobName. After constructing the request,
+ * users may optionally specify object metadata or a canned ACL as well.
*
- * @param bucketName
- * @param filePath
- * @param fullfilePath
+ * @param bucketName The name of an existing bucket to which the new object will be
+ * uploaded.
+ * @param jobName The jobName under which to store the new object.
+ * @param file The path of the file to upload to Amazon S3.
*/
- public void uploadFileToBucket(String bucketName, String filePath, String fullfilePath) {
- PutObjectRequest request = new PutObjectRequest(bucketName, filePath, new File(fullfilePath));
+ private void uploadFileToBucket(String bucketName, String jobName, String file) {
+ PutObjectRequest request = new PutObjectRequest(bucketName, jobName, new File(file));
amazonS3.putObject(request);
}
- /**
- * Gets Transcription result from AWS S3 bucket given bucketName and key
- *
- * @param key
- * @return
- */
- public String getTranscriptResult(String key) {
- TranscriptionJob transcriptionJob = retrieveObjectWhenJobCompleted(key);
- if (transcriptionJob != null && !TranscriptionJobStatus.FAILED.equals(transcriptionJob.getTranscriptionJobStatus())) {
- return amazonS3.getObjectAsString(this.bucketName, key + ".json");
- } else
- return null;
- }
-
- /**
- * Private helper function to get object from s3
- *
- * @param key
- * @return
- */
- private TranscriptionJob retrieveObjectWhenJobCompleted(String key) {
- GetTranscriptionJobRequest getTranscriptionJobRequest = new GetTranscriptionJobRequest();
- getTranscriptionJobRequest.setTranscriptionJobName(key);
- while (true) {
- GetTranscriptionJobResult innerResult = amazonTranscribe.getTranscriptionJob(getTranscriptionJobRequest);
- String status = innerResult.getTranscriptionJob().getTranscriptionJobStatus();
- if (TranscriptionJobStatus.COMPLETED.name().equals(status) ||
- TranscriptionJobStatus.FAILED.name().equals(status)) {
- return innerResult.getTranscriptionJob();
- }
- }
- }
-
/**
* @param filePath
* @throws TikaException
@@ -121,6 +92,7 @@ public class AmazonTranscribe implements Transcriber {
*/
@Override
public String startTranscribeAudio(String filePath) throws TikaException, IOException {
+ uploadFileToBucket(this.bucketName, this.jobName, filePath);
if (!isAvailable) return "";
StartTranscriptionJobRequest startTranscriptionJobRequest = new StartTranscriptionJobRequest();
Media media = new Media();
@@ -130,11 +102,13 @@ public class AmazonTranscribe implements Transcriber {
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
- return getTranscriptResult(jobName + ".json");
+// return getTranscriptResult(jobName + ".json");
+ return getTranscriptResult(jobName);
}
@Override
public String startTranscribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException {
+ uploadFileToBucket(this.bucketName, this.jobName, filePath);
if (!isAvailable) return "";
StartTranscriptionJobRequest startTranscriptionJobRequest = new StartTranscriptionJobRequest();
Media media = new Media();
@@ -144,11 +118,13 @@ public class AmazonTranscribe implements Transcriber {
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
- return getTranscriptResult(jobName + ".json");
+// return getTranscriptResult(jobName + ".json");
+ return getTranscriptResult(jobName);
}
@Override
public String startTranscribeVideo(String filePath) throws TikaException, IOException {
+ uploadFileToBucket(this.bucketName, this.jobName, filePath);
if (!isAvailable) return "";
//TODO
return "";
@@ -156,6 +132,7 @@ public class AmazonTranscribe implements Transcriber {
@Override
public String startTranscribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException {
+ uploadFileToBucket(this.bucketName, this.jobName, filePath);
if (!isAvailable) return "";
//TODO
return "";
@@ -204,4 +181,37 @@ public class AmazonTranscribe implements Transcriber {
bucketName != null &&
!bucketName.equals(DEFAULT_BUCKET);
}
+
+ /**
+ * Gets Transcription result from AWS S3 bucket given bucketName and jobName
+ *
+ * @param jobName
+ * @return
+ */
+ public String getTranscriptResult(String jobName) {
+ TranscriptionJob transcriptionJob = retrieveObjectWhenJobCompleted(jobName);
+ if (transcriptionJob != null && !TranscriptionJobStatus.FAILED.equals(transcriptionJob.getTranscriptionJobStatus())) {
+ return amazonS3.getObjectAsString(this.bucketName, jobName);
+ } else
+ return null;
+ }
+
+ /**
+ * Private helper function to get object from s3
+ *
+ * @param jobName
+ * @return
+ */
+ private TranscriptionJob retrieveObjectWhenJobCompleted(String jobName) {
+ GetTranscriptionJobRequest getTranscriptionJobRequest = new GetTranscriptionJobRequest();
+ getTranscriptionJobRequest.setTranscriptionJobName(jobName);
+ while (true) {
+ GetTranscriptionJobResult innerResult = amazonTranscribe.getTranscriptionJob(getTranscriptionJobRequest);
+ String status = innerResult.getTranscriptionJob().getTranscriptionJobStatus();
+ if (TranscriptionJobStatus.COMPLETED.name().equals(status) ||
+ TranscriptionJobStatus.FAILED.name().equals(status)) {
+ return innerResult.getTranscriptionJob();
+ }
+ }
+ }
}


message:
added documentation.
Made the interface AWS independent some other small fixes.

diff:
@@ -31,21 +31,22 @@ public interface Transcriber {
/**
* Transcribe the given audio file.
*
- * @param filePath
- * @return
- * @throws TikaException When there is an error translating.
+ * @param filePath The path of the file to be transcribed.
+ * @return key for transcription lookup
+ * @throws TikaException When there is an error transcribing.
* @throws IOException
* @since TODO
*/
+
public String startTranscribeAudio(String filePath) throws TikaException, IOException;
/**
* Transcribe the given the audio file and the source language.
*
- * @param filePath
- * @param sourceLanguage
- * @return
- * @throws TikaException When there is an error translating.
+ * @param filePath The path of the file to be transcribed.
+ * @param sourceLanguage The language code for the language used in the input media file.
+ * @return key for transcription lookup
+ * @throws TikaException When there is an error transcribing.
* @throws java.io.IOException
* @since TODO
*/
@@ -54,9 +55,9 @@ public interface Transcriber {
/**
* Transcribe the given the video file.
*
- * @param filePath
- * @return
- * @throws TikaException When there is an error translating.
+ * @param filePath The path of the file to be transcribed.
+ * @return key for transcription lookup
+ * @throws TikaException When there is an error transcribing.
* @throws java.io.IOException
* @since TODO
*/
@@ -65,10 +66,10 @@ public interface Transcriber {
/**
* Transcribe the given the video file and the source language.
*
- * @param filePath
- * @param sourceLanguage
- * @return
- * @throws TikaException When there is an error translating.
+ * @param filePath The path of the file to be transcribed.
+ * @param sourceLanguage The language code for the language used in the input media file.
+ * @return key for transcription lookup
+ * @throws TikaException When there is an error transcribing.
* @throws java.io.IOException
* @since TODO
*/
@@ -79,11 +80,4 @@ public interface Transcriber {
* @since Tika TODO
*/
public boolean isAvailable();
-
- /**
- * Returns transcription result using specific key
- * @param fileNameS3
- * @return transcribed text
- */
- public String getTranscriptResult(String fileNameS3);
}

diff:
@@ -69,8 +69,7 @@ public class AmazonTranscribe implements Transcriber {
}
private String getJobKey() {
- UUID uuid = UUID.randomUUID();
- return uuid.toString();
+ return UUID.randomUUID().toString();
}
/**
@@ -78,7 +77,9 @@ public class AmazonTranscribe implements Transcriber {
* {@link PutObjectRequest} object to upload a file to the
* specified bucket and jobName. After constructing the request,
* users may optionally specify object metadata or a canned ACL as well.
+ *
* @param filePath The path of the file to upload to Amazon S3.
+ * @param jobName The unique job name for each job(UUID).
*/
private void uploadFileToBucket(String filePath, String jobName) {
PutObjectRequest request = new PutObjectRequest(this.bucketName, jobName, new File(filePath));
@@ -87,9 +88,10 @@ public class AmazonTranscribe implements Transcriber {
/**
* Starts AWS Transcribe Job without language specification for Audio
- * @param filePath
+ *
+ * @param filePath The path of the file to upload to Amazon S3.
* @return key for transcription lookup
- * @throws TikaException
+ * @throws TikaException When there is an error transcribing.
* @throws IOException
*/
@Override
@@ -106,13 +108,16 @@ public class AmazonTranscribe implements Transcriber {
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
return jobName;
}
+
/**
* Starts AWS Transcribe Job with language specification for Audio
- * @param filePath
- * @param sourceLanguage
+ *
+ * @param filePath The path of the file to upload to Amazon S3.
+ * @param sourceLanguage The language code for the language used in the input media file
* @return key for transcription lookup
- * @throws TikaException
+ * @throws TikaException When there is an error transcribing.
* @throws IOException
+ * @see AWS Language Code
*/
@Override
public String startTranscribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException {
@@ -123,7 +128,7 @@ public class AmazonTranscribe implements Transcriber {
Media media = new Media();
media.setMediaFileUri(amazonS3.getUrl(bucketName, filePath).toString());
startTranscriptionJobRequest.withMedia(media)
- .withLanguageCode(LanguageCode.valueOf(sourceLanguage))
+ .withLanguageCode(LanguageCode.fromValue(sourceLanguage))
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
@@ -132,9 +137,10 @@ public class AmazonTranscribe implements Transcriber {
/**
* Starts AWS Transcribe Job without language specification for Video
- * @param filePath
+ *
+ * @param filePath The path of the file to upload to Amazon S3.
* @return key for transcription lookup
- * @throws TikaException
+ * @throws TikaException When there is an error transcribing.
* @throws IOException
*/
@Override
@@ -154,11 +160,13 @@ public class AmazonTranscribe implements Transcriber {
/**
* Starts AWS Transcribe Job with language specification for Audio
- * @param filePath
- * @param sourceLanguage
+ *
+ * @param filePath The path of the file to upload to Amazon S3.
+ * @param sourceLanguage The language code for the language used in the input media file.
* @return key for transcription lookup
- * @throws TikaException
+ * @throws TikaException When there is an error transcribing.
* @throws IOException
+ * @see AWS Language Code
*/
@Override
public String startTranscribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException {
@@ -169,7 +177,7 @@ public class AmazonTranscribe implements Transcriber {
Media media = new Media();
media.setMediaFileUri(amazonS3.getUrl(bucketName, filePath).toString());
startTranscriptionJobRequest.withMedia(media)
- .withLanguageCode(LanguageCode.valueOf(sourceLanguage))
+ .withLanguageCode(LanguageCode.fromValue(sourceLanguage))
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
@@ -223,10 +231,9 @@ public class AmazonTranscribe implements Transcriber {
/**
* Gets Transcription result from AWS S3 bucket given bucketName and jobName
*
- * @param fileNameS3
- * @return
+ * @param fileNameS3 The path of the file to upload to Amazon S3.
+ * @return The transcribed result.
*/
- @Override
public String getTranscriptResult(String fileNameS3) {
TranscriptionJob transcriptionJob = retrieveObjectWhenJobCompleted(fileNameS3);
if (transcriptionJob != null && !TranscriptionJobStatus.FAILED.equals(transcriptionJob.getTranscriptionJobStatus())) {
@@ -238,8 +245,8 @@ public class AmazonTranscribe implements Transcriber {
/**
* Private helper function to get object from s3
*
- * @param jobName
- * @return
+ * @param jobName The unique job name for each job(UUID).
+ * @return TranscriptionJob object
*/
private TranscriptionJob retrieveObjectWhenJobCompleted(String jobName) {
GetTranscriptionJobRequest getTranscriptionJobRequest = new GetTranscriptionJobRequest();


message: no need for support of overhead conversion of mp4 to mp3

diff:
@@ -145,17 +145,7 @@ public class AmazonTranscribe implements Transcriber {
*/
@Override
public String startTranscribeVideo(String filePath) throws TikaException, IOException {
- if (!isAvailable()) return null;
- String jobName = getJobKey();
- uploadFileToBucket(filePath, jobName);
- StartTranscriptionJobRequest startTranscriptionJobRequest = new StartTranscriptionJobRequest();
- Media media = new Media();
- media.setMediaFileUri(amazonS3.getUrl(bucketName, filePath).toString());
- startTranscriptionJobRequest.withMedia(media)
- .withOutputBucketName(this.bucketName)
- .setTranscriptionJobName(jobName);
- amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
- return jobName;
+ return startTranscribeAudio(filePath);
}
/**
@@ -170,18 +160,7 @@ public class AmazonTranscribe implements Transcriber {
*/
@Override
public String startTranscribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException {
- if (!isAvailable()) return null;
- String jobName = getJobKey();
- uploadFileToBucket(filePath, jobName);
- StartTranscriptionJobRequest startTranscriptionJobRequest = new StartTranscriptionJobRequest();
- Media media = new Media();
- media.setMediaFileUri(amazonS3.getUrl(bucketName, filePath).toString());
- startTranscriptionJobRequest.withMedia(media)
- .withLanguageCode(LanguageCode.fromValue(sourceLanguage))
- .withOutputBucketName(this.bucketName)
- .setTranscriptionJobName(jobName);
- amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
- return jobName;
+ return startTranscribeAudio(filePath, sourceLanguage);
}
@Override


message: few fixes based off comments

diff:
@@ -24,8 +24,9 @@ import java.io.IOException;
/**
* Interface for Transcriber services.
+ * TODO add documentation
*
- * @since Tika TODO
+ * @since Tika 2.1
*/
public interface Transcriber {
/**
@@ -34,11 +35,11 @@ public interface Transcriber {
* @param filePath The path of the file to be transcribed.
* @return key for transcription lookup
* @throws TikaException When there is an error transcribing.
- * @throws IOException
- * @since TODO
+ * @throws IOException If an I/O exception of some sort has occurred.
+ * @since 2.1
*/
- public String startTranscribeAudio(String filePath) throws TikaException, IOException;
+ public String transcribeAudio(String filePath) throws TikaException, IOException;
/**
* Transcribe the given the audio file and the source language.
@@ -47,10 +48,10 @@ public interface Transcriber {
* @param sourceLanguage The language code for the language used in the input media file.
* @return key for transcription lookup
* @throws TikaException When there is an error transcribing.
- * @throws java.io.IOException
- * @since TODO
+ * @throws IOException If an I/O exception of some sort has occurred.
+ * @since 2.1
*/
- public String startTranscribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException;
+ public String transcribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException;
/**
* Transcribe the given the video file.
@@ -58,10 +59,10 @@ public interface Transcriber {
* @param filePath The path of the file to be transcribed.
* @return key for transcription lookup
* @throws TikaException When there is an error transcribing.
- * @throws java.io.IOException
- * @since TODO
+ * @throws IOException If an I/O exception of some sort has occurred.
+ * @since 2.1
*/
- public String startTranscribeVideo(String filePath) throws TikaException, IOException;
+ public String transcribeVideo(String filePath) throws TikaException, IOException;
/**
* Transcribe the given the video file and the source language.
@@ -70,14 +71,14 @@ public interface Transcriber {
* @param sourceLanguage The language code for the language used in the input media file.
* @return key for transcription lookup
* @throws TikaException When there is an error transcribing.
- * @throws java.io.IOException
- * @since TODO
+ * @throws IOException If an I/O exception of some sort has occurred.
+ * @since 2.1
*/
- public String startTranscribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException;
+ public String transcribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException;
/**
* @return true if this Transcriber is probably able to translate right now.
- * @since Tika TODO
+ * @since Tika 2.1
*/
public boolean isAvailable();
}

diff:
@@ -20,7 +20,13 @@ package org.apache.tika.transcribe;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.transcribe.AmazonTranscribeAsync;
-import com.amazonaws.services.transcribe.model.*;
+import com.amazonaws.services.transcribe.model.GetTranscriptionJobRequest;
+import com.amazonaws.services.transcribe.model.Media;
+import com.amazonaws.services.transcribe.model.StartTranscriptionJobRequest;
+import com.amazonaws.services.transcribe.model.TranscriptionJob;
+import com.amazonaws.services.transcribe.model.GetTranscriptionJobResult;
+import com.amazonaws.services.transcribe.model.TranscriptionJobStatus;
+import com.amazonaws.services.transcribe.model.LanguageCode;
import org.apache.tika.exception.TikaException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,7 +36,9 @@ import java.io.IOException;
import java.util.Properties;
import java.util.UUID;
-
+/**
+ * TODO Add description about required configuration and any specific characteristics for this Transcriber.
+ */
public class AmazonTranscribe implements Transcriber {
public static final String PROPERTIES_FILE = "transcribe.amazon.properties";
@@ -48,8 +56,6 @@ public class AmazonTranscribe implements Transcriber {
private boolean isAvailable; // Flag for whether or not translation is available.
private String clientId;
private String clientSecret; // Keys used for the API calls.
-// private HashSet<String> validSourceLanguages = new HashSet<>(Arrays.asList("en-US", "en-GB", "es-US", "fr-CA", "fr-FR", "en-AU",
-// "it-IT", "de-DE", "pt-BR", "ja-JP", "ko-KR")); // Valid inputs to StartStreamTranscription for language of source file (audio)
public AmazonTranscribe() {
Properties config = new Properties();
@@ -92,10 +98,10 @@ public class AmazonTranscribe implements Transcriber {
* @param filePath The path of the file to upload to Amazon S3.
* @return key for transcription lookup
* @throws TikaException When there is an error transcribing.
- * @throws IOException
+ * @throws IOException If an I/O exception of some sort has occurred.
*/
@Override
- public String startTranscribeAudio(String filePath) throws TikaException, IOException {
+ public String transcribeAudio(String filePath) throws TikaException, IOException {
if (!isAvailable()) return null;
String jobName = getJobKey();
uploadFileToBucket(filePath, jobName);
@@ -105,7 +111,11 @@ public class AmazonTranscribe implements Transcriber {
startTranscriptionJobRequest.withMedia(media)
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
+ try {
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
+ } catch (Exception e) {
+ throw new TikaException("Error occurred while AWS transcribing.: " + e.getMessage());
+ }
return jobName;
}
@@ -116,11 +126,11 @@ public class AmazonTranscribe implements Transcriber {
* @param sourceLanguage The language code for the language used in the input media file
* @return key for transcription lookup
* @throws TikaException When there is an error transcribing.
- * @throws IOException
+ * @throws IOException If an I/O exception of some sort has occurred.
* @see AWS Language Code
*/
@Override
- public String startTranscribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException {
+ public String transcribeAudio(String filePath, String sourceLanguage) throws TikaException, IOException {
if (!isAvailable()) return null;
String jobName = getJobKey();
uploadFileToBucket(filePath, jobName);
@@ -131,7 +141,11 @@ public class AmazonTranscribe implements Transcriber {
.withLanguageCode(LanguageCode.fromValue(sourceLanguage))
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
+ try {
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
+ } catch (Exception e) {
+ throw new TikaException("Error occurred while AWS transcribing.: " + e.getMessage());
+ }
return jobName;
}
@@ -141,10 +155,10 @@ public class AmazonTranscribe implements Transcriber {
* @param filePath The path of the file to upload to Amazon S3.
* @return key for transcription lookup
* @throws TikaException When there is an error transcribing.
- * @throws IOException
+ * @throws IOException If an I/O exception of some sort has occurred.
*/
@Override
- public String startTranscribeVideo(String filePath) throws TikaException, IOException {
+ public String transcribeVideo(String filePath) throws TikaException, IOException {
if (!isAvailable()) return null;
String jobName = getJobKey();
uploadFileToBucket(filePath, jobName);
@@ -154,7 +168,11 @@ public class AmazonTranscribe implements Transcriber {
startTranscriptionJobRequest.withMedia(media)
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
+ try {
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
+ } catch (Exception e) {
+ throw new TikaException("Error occurred while AWS transcribing.: " + e.getMessage());
+ }
return jobName;
}
@@ -165,11 +183,11 @@ public class AmazonTranscribe implements Transcriber {
* @param sourceLanguage The language code for the language used in the input media file.
* @return key for transcription lookup
* @throws TikaException When there is an error transcribing.
- * @throws IOException
+ * @throws IOException If an I/O exception of some sort has occurred.
* @see AWS Language Code
*/
@Override
- public String startTranscribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException {
+ public String transcribeVideo(String filePath, String sourceLanguage) throws TikaException, IOException {
if (!isAvailable()) return null;
String jobName = getJobKey();
uploadFileToBucket(filePath, jobName);
@@ -180,7 +198,11 @@ public class AmazonTranscribe implements Transcriber {
.withLanguageCode(LanguageCode.fromValue(sourceLanguage))
.withOutputBucketName(this.bucketName)
.setTranscriptionJobName(jobName);
+ try {
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
+ } catch (Exception e) {
+ throw new TikaException("Error occurred while AWS transcribing.: " + e.getMessage());
+ }
return jobName;
}

diff:
@@ -42,7 +42,7 @@ public class AmazonTranscribeGuessLanguageTest {
if (transcriber.isAvailable()) {
try {
- jobName = transcriber.startTranscribeAudio(audioFilePath);
+ jobName = transcriber.transcribeAudio(audioFilePath);
result = transcriber.getTranscriptResult(jobName);
assertNotNull(result);
assertEquals("Result: [" + result
@@ -65,7 +65,7 @@ public class AmazonTranscribeGuessLanguageTest {
if (transcriber.isAvailable()) {
try {
- jobName = transcriber.startTranscribeAudio(audioFilePath);
+ jobName = transcriber.transcribeAudio(audioFilePath);
result = transcriber.getTranscriptResult(jobName);
assertNotNull(result);
assertEquals("Result: [" + result
@@ -88,7 +88,7 @@ public class AmazonTranscribeGuessLanguageTest {
if (transcriber.isAvailable()) {
try {
- jobName = transcriber.startTranscribeAudio(videoFilePath);
+ jobName = transcriber.transcribeAudio(videoFilePath);
result = transcriber.getTranscriptResult(jobName);
assertNotNull(result);
assertEquals("Result: [" + result
@@ -111,7 +111,7 @@ public class AmazonTranscribeGuessLanguageTest {
if (transcriber.isAvailable()) {
try {
- jobName = transcriber.startTranscribeAudio(videoFilePath);
+ jobName = transcriber.transcribeAudio(videoFilePath);
result = transcriber.getTranscriptResult(jobName);
assertNotNull(result);
assertEquals("Result: [" + result

diff:
*/
package org.apache.tika.transcribe;
-import org.apache.tika.transcribe.AmazonTranscribe;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@@ -42,7 +41,7 @@ public class AmazonTranscribeSimpleTest {
String result = null;
if (transcribe.isAvailable()) {
try {
- result = transcribe.startTranscribeAudio(source, "en-GB");
+ result = transcribe.transcribeAudio(source, "en-GB");
assertNotNull(result);
assertEquals("Result: [" + result
+ "]: not equal to expected: [" + expected + "]",
@@ -63,7 +62,7 @@ public class AmazonTranscribeSimpleTest {
String result = null;
if (transcribe.isAvailable()) {
try {
- result = transcribe.startTranscribeAudio(source, "fr-FR");
+ result = transcribe.transcribeAudio(source, "fr-FR");
assertNotNull(result);
assertEquals("Result: [" + result
+ "]: not equal to expected: [" + expected + "]",
@@ -84,7 +83,7 @@ public class AmazonTranscribeSimpleTest {
String result = null;
if (transcribe.isAvailable()) {
try {
- result = transcribe.startTranscribeVideo(source, "en");
+ result = transcribe.transcribeVideo(source, "en");
assertNotNull(result);
assertEquals("Result: [" + result
+ "]: not equal to expected: [" + expected + "]",
@@ -106,7 +105,7 @@ public class AmazonTranscribeSimpleTest {
String result = null;
if (transcribe.isAvailable()) {
try {
- result = transcribe.startTranscribeVideo(source, "ko-KR");
+ result = transcribe.transcribeVideo(source, "ko-KR");
assertNotNull(result);
assertEquals("Result: [" + result
+ "]: not equal to expected: [" + expected + "]",

original_message: few fixes based off comments


message: white spaces and fixes

diff:
@@ -21,7 +21,6 @@ import org.apache.tika.exception.TikaException;
import java.io.IOException;
-
/**
* Interface for Transcriber services.
* TODO add documentation
@@ -52,15 +51,6 @@ public interface Transcriber {
*/
public String transcribe(String filePath, String sourceLanguage) throws TikaException, IOException;
- /**
- * Gets Transcription result from AWS S3 bucket given bucketName and jobName
- *
- * @param key
- * @return
- */
- public String getTranscriptResult(String key);
-
-
/**
* @return true if this Transcriber is probably able to translate right now.
* @since Tika 2.1

diff:
package org.apache.tika.transcribe;
-import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.PutObjectRequest;
@@ -33,7 +32,6 @@ import java.io.IOException;
import java.util.Properties;
import java.util.UUID;
-
public class AmazonTranscribe implements Transcriber {
public static final String PROPERTIES_FILE = "transcribe.amazon.properties";
@@ -43,7 +41,6 @@ public class AmazonTranscribe implements Transcriber {
public static final String DEFAULT_SECRET = "dummy-secret";
public static final String DEFAULT_BUCKET = "dummy-bucket";
public static final String BUCKET_NAME = "transcribe.BUCKET_NAME";
-
private static final Logger LOG = LoggerFactory.getLogger(AmazonTranscribe.class);
private AmazonTranscribeAsync amazonTranscribe;
private AmazonS3 amazonS3;
@@ -51,8 +48,6 @@ public class AmazonTranscribe implements Transcriber {
private boolean isAvailable; // Flag for whether or not translation is available.
private String clientId;
private String clientSecret; // Keys used for the API calls.
-// private HashSet<String> validSourceLanguages = new HashSet<>(Arrays.asList("en-US", "en-GB", "es-US", "fr-CA", "fr-FR", "en-AU",
-// "it-IT", "de-DE", "pt-BR", "ja-JP", "ko-KR")); // Valid inputs to StartStreamTranscription for language of source file (audio)
public AmazonTranscribe() {
Properties config = new Properties();
@@ -72,8 +67,7 @@ public class AmazonTranscribe implements Transcriber {
}
private String getJobKey() {
- UUID uuid = UUID.randomUUID();
- return uuid.toString();
+ return UUID.randomUUID().toString();
}
/**
@@ -81,6 +75,7 @@ public class AmazonTranscribe implements Transcriber {
* {@link PutObjectRequest} object to upload a file to the
* specified bucket and jobName. After constructing the request,
* users may optionally specify object metadata or a canned ACL as well.
+ *
* @param filePath The path of the file to upload to Amazon S3.
*/
private void uploadFileToBucket(String filePath, String jobName) throws TikaException {
@@ -91,12 +86,11 @@ public class AmazonTranscribe implements Transcriber {
} catch (SdkClientException e) {
throw (new TikaException("File Upload to AWS Failed"));
}
-
-
}
/**
* Starts AWS Transcribe Job without language specification
+ *
* @param filePath
* @return key for transcription lookup
* @throws TikaException
@@ -116,8 +110,10 @@ public class AmazonTranscribe implements Transcriber {
amazonTranscribe.startTranscriptionJob(startTranscriptionJobRequest);
return jobName;
}
+
/**
* Starts AWS Transcribe Job with language specification
+ *
* @param filePath
* @param sourceLanguage
* @return key for transcription lookup
@@ -140,7 +136,6 @@ public class AmazonTranscribe implements Transcriber {
return jobName;
}
-
@Override
public boolean isAvailable() {
return this.isAvailable;
@@ -191,7 +186,6 @@ public class AmazonTranscribe implements Transcriber {
* @param fileNameS3
* @return
*/
- @Override
public String getTranscriptResult(String fileNameS3) {
TranscriptionJob transcriptionJob = retrieveObjectWhenJobCompleted(fileNameS3);
if (transcriptionJob != null && !TranscriptionJobStatus.FAILED.equals(transcriptionJob.getTranscriptionJobStatus())) {

original_message: white spaces and fixes


diff:
package org.apache.tika.detect;
import java.io.ByteArrayInputStream;
+import java.io.CharConversionException;
import java.io.InputStream;
+import java.util.Arrays;
import javax.xml.namespace.QName;
import org.apache.commons.io.input.CloseShieldInputStream;
@@ -39,20 +41,45 @@ public class XmlRootExtractor {
private static final ParseContext EMPTY_CONTEXT = new ParseContext();
public QName extractRootElement(byte[] data) {
- return extractRootElement(new ByteArrayInputStream(data));
+ // this loop should be very rare
+ while (true) {
+ try {
+ return extractRootElement(new ByteArrayInputStream(data), true);
+ } catch (MalformedCharException e) {
+ // see TIKA-3596, try to handle truncated/bad encoded XML files
+ int newLen = data.length / 2;
+ if (newLen % 2 == 1) {
+ newLen--;
+ }
+ if (newLen > 0) {
+ data = Arrays.copyOf(data, newLen);
+ } else {
+ break;
+ }
+ }
+ }
+ return null;
}
/**
* @since Apache Tika 0.9
*/
public QName extractRootElement(InputStream stream) {
+ return extractRootElement(stream, false);
+ }
+
+ private QName extractRootElement(InputStream stream, boolean throwMalformed) {
ExtractorHandler handler = new ExtractorHandler();
try {
XMLReaderUtils.parseSAX(new CloseShieldInputStream(stream),
new OfflineContentHandler(handler), EMPTY_CONTEXT);
} catch (SecurityException e) {
throw e;
- } catch (Exception ignore) {
+ } catch (Exception e) {
+ if (throwMalformed && (e instanceof CharConversionException
+ || e.getCause() instanceof CharConversionException)) {
+ throw new MalformedCharException(e);
+ }
}
return handler.rootElement;
}
@@ -70,4 +97,12 @@ public class XmlRootExtractor {
}
+ private static class MalformedCharException extends RuntimeException {
+
+ public MalformedCharException(Exception e) {
+ super(e);
+ }
+
+ }
+
}

diff:
@@ -78,6 +78,9 @@ public class MimeDetectionTest {
//test GCMD Directory Interchange Format (.dif) TIKA-1561
testFile("application/dif+xml", "brwNIMS_2014.dif");
+
+ // truncated xml should still be detected as xml, See TIKA-3596
+ testFile("application/xml", "truncated-utf16-xml.xyz");
}
@Test

diff: Binary files /dev/null and b/tika-core/src/test/resources/org/apache/tika/mime/truncated-utf16-xml.xyz differ

original_message: TIKA-3596: detect truncated/bad encoded xml files as application/xml


diff:
<match value="\nUser-Agent:" type="string" offset="0:1024"/>
<match value="\nX-Mailer:" type="string" offset="0:1024"/>
<match value="\nX-Originating-IP:" type="stringignorecase" offset="0:1024"/>
- <match value="\nX-" type="string" offset="0:1024"/>
<match value="\nDKIM-" type="string" offset="0:1024"/>
<match value="\nARC-" type="string" offset="0:1024"/>
</match>

diff:
@@ -260,4 +260,12 @@ public class MimeDetectionTest {
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "testingTESTINGtesting");
assertEquals(helloXType, MIME_TYPES.detect(new ByteArrayInputStream(helloWorld), metadata));
}
+
+ /**
+ * Test for TIKA-3771.
+ */
+ @Test
+ public void testPNGWithSomeEmlHeaders() throws IOException {
+ testFile("image/png", "test-pngNotEml.bin");
+ }
}

diff: Binary files /dev/null and b/tika-core/src/test/resources/org/apache/tika/mime/test-pngNotEml.bin differ

original_message: TIKA-3771: remove eml magic too common causing false positives


diff:
@@ -75,7 +75,7 @@ public class Icu4jEncodingDetector implements EncodingDetector {
try {
String n = match.getNormalizedName();
if (ignoreCharsets.contains(n)) {
- return null;
+ continue;
}
return CharsetUtils.forName(match.getNormalizedName());
} catch (IllegalArgumentException e) {

diff:
@@ -136,10 +136,16 @@ public class CharsetDetectorTest extends TikaTest {
TikaConfig tikaConfig = new TikaConfig(
getResourceAsStream("/test-configs/tika-config-ignore-charset.xml"));
- Metadata m = new Metadata();
+ AutoDetectParser parser = new AutoDetectParser(tikaConfig);
+ Metadata m = new Metadata();
m.set(TikaCoreProperties.RESOURCE_NAME_KEY, "texty-text.txt");
assertContains("ACTIVE AGE", getXML("testIgnoreCharset.txt",
- new AutoDetectParser(tikaConfig), m).xml);
+ parser, m).xml);
+
+ m = new Metadata();
+ m.set(TikaCoreProperties.RESOURCE_NAME_KEY, "texty-text.txt");
+ assertContains("Please check your email", getXML("test_ignore_IBM420.html",
+ parser, m).xml);
}
}

diff:
<encodingDetector class="org.apache.tika.parser.txt.Icu4jEncodingDetector">
<params>
<param name="ignoreCharsets" type="list">
+ <string>IBM420</string>
<string>IBM424</string>
</param>
</params>

diff: Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/test_ignore_IBM420.html differ

original_message: TIKA-3774: fix ignoreCharsets param of Icu4jEncodingDetector


diff:
@@ -499,8 +499,8 @@ public class TestMetadata {
start += random.nextInt(1000000);
Date now = new Date(start);
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
- m.set(TikaCoreProperties.CREATED, df.format(now));
df.setTimeZone(TimeZone.getTimeZone("UTC"));
+ m.set(TikaCoreProperties.CREATED, df.format(now));
assertTrue(
Math.abs(now.getTime() - m.getDate(TikaCoreProperties.CREATED).getTime()) <
2000);

diff:
<version>${maven.surefire.version}
</version> <!-- versions greater than this don't like System.exit calls in tika-batch -->
<configuration>
- <argLine>-Xmx3072m -Duser.timezone=UTC -Djava.awt.headless=true</argLine>
+ <argLine>-Xmx3072m -Djava.awt.headless=true</argLine>
</configuration>
</plugin>
<plugin>

diff:
@@ -26,6 +26,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.Locale;
+import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -377,8 +378,15 @@ public class ImageMetadataExtractor {
static class ExifHandler implements DirectoryHandler {
// There's a new ExifHandler for each file processed, so this is thread safe
- private final SimpleDateFormat dateUnspecifiedTz =
- new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US);
+ private final SimpleDateFormat dateUnspecifiedTz = getUnspecifiedTzDateFormat();
+
+ private SimpleDateFormat getUnspecifiedTzDateFormat() {
+ SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US);
+ // As of Drew Noakes' metadata-extractor 2.8.1, unspecified
+ // timezones are set to TimeZone.getTimeZone("GMT")
+ df.setTimeZone(TimeZone.getTimeZone("GMT"));
+ return df;
+ }
public boolean supports(Class<? extends Directory> directoryType) {

diff:
@@ -73,11 +73,11 @@ public class ImageMetadataExtractorTest {
public void testExifHandlerParseDate() throws MetadataException {
ExifSubIFDDirectory exif = Mockito.mock(ExifSubIFDDirectory.class);
Mockito.when(exif.containsTag(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL)).thenReturn(true);
- GregorianCalendar calendar = new GregorianCalendar(TimeZone.getDefault(), Locale.ROOT);
+ GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT);
calendar.setTimeInMillis(0);
calendar.set(2000, 0, 1, 0, 0, 0);
Mockito.when(exif.getDate(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL))
- .thenReturn(calendar.getTime()); // jvm default timezone as in Metadata Extractor
+ .thenReturn(calendar.getTime()); // UTC timezone as in Metadata Extractor
Metadata metadata = new Metadata();
new ImageMetadataExtractor.ExifHandler().handle(exif, metadata);
@@ -89,11 +89,11 @@ public class ImageMetadataExtractorTest {
public void testExifHandlerParseDateFallback() throws MetadataException {
ExifIFD0Directory exif = Mockito.mock(ExifIFD0Directory.class);
Mockito.when(exif.containsTag(ExifIFD0Directory.TAG_DATETIME)).thenReturn(true);
- GregorianCalendar calendar = new GregorianCalendar(TimeZone.getDefault(), Locale.ROOT);
+ GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT);
calendar.setTimeInMillis(0);
calendar.set(1999, 0, 1, 0, 0, 0);
Mockito.when(exif.getDate(ExifIFD0Directory.TAG_DATETIME))
- .thenReturn(calendar.getTime()); // jvm default timezone as in Metadata Extractor
+ .thenReturn(calendar.getTime()); // UTC timezone as in Metadata Extractor
Metadata metadata = new Metadata();
new ImageMetadataExtractor.ExifHandler().handle(exif, metadata);

diff:
@@ -26,8 +26,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.TimeZone;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.xml.sax.helpers.DefaultHandler;
@@ -43,22 +41,6 @@ public class JpegParserTest {
static TimeZone CURR_TIME_ZONE = TimeZone.getDefault();
private final Parser parser = new JpegParser();
- //As of Drew Noakes' metadata-extractor 2.8.1,
- //unspecified timezones appear to be set to
- //TimeZone.getDefault(). We need to normalize this
- //for testing across different time zones.
- //We also appear to have to specify it in the surefire config:
- //<argLine>-Duser.timezone=UTC</argLine>
- @BeforeAll
- public static void setDefaultTimeZone() {
- TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
- }
-
- @AfterAll
- public static void resetDefaultTimeZone() {
- TimeZone.setDefault(CURR_TIME_ZONE);
- }
-
@Test
public void testJPEG() throws Exception {
Metadata metadata = new Metadata();

original_message:
TIKA-3815: set GMT timezone for unspecified timezones like drew noakes,
fix tests to work on different timezones


message: fix RFC822ParserTest failing tests

diff:
@@ -38,6 +38,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Locale;
+import java.util.TimeZone;
import org.apache.james.mime4j.stream.MimeConfig;
import org.junit.jupiter.api.BeforeAll;
@@ -380,20 +381,23 @@ public class RFC822ParserTest extends TikaTest {
String expected = "2016-05-15T01:32:00Z";
- for (String dateString : new String[]{"Sun, 15 May 2016 01:32:00 UTC",
- //make sure this test basically works
- "Sun, 15 May 2016 01:32:00", //no timezone
- "Sunday, May 15 2016 1:32 AM", "May 15 2016 1:32am", "May 15 2016 1:32 am",
- "2016-05-15 01:32:00", " Sun, 15 May 2016 3:32:00 +0200",
+ int dateNum = 0;
+ for (String dateString : new String[] {
+ // with timezone info:
+ "Sun, 15 May 2016 01:32:00 UTC", " Sun, 15 May 2016 3:32:00 +0200",
// format correctly handled by mime4j if no leading whitespace
- " Sun, 14 May 2016 20:32:00 EST",}) {
- testDate(dateString, expected);
+ " Sun, 14 May 2016 20:32:00 EST",
+ // no timezone info:
+ "Sun, 15 May 2016 01:32:00",
+ "Sunday, May 15 2016 1:32 AM", "May 15 2016 1:32am", "May 15 2016 1:32 am",
+ "2016-05-15 01:32:00", }) {
+ testDate(dateString, expected, dateNum++ < 3);
}
//now try days without times
expected = "2016-05-15T12:00:00Z";
for (String dateString : new String[]{"May 15, 2016", "Sun, 15 May 2016", "15 May 2016",}) {
- testDate(dateString, expected);
+ testDate(dateString, expected, true);
}
}
@@ -419,11 +423,14 @@ public class RFC822ParserTest extends TikaTest {
}
- private void testDate(String dateString, String expected) throws Exception {
+ private void testDate(String dateString, String expected, boolean useUTC) throws Exception {
Date parsedDate = getDate(dateString);
assertNotNull(parsedDate, "couldn't parse " + dateString);
DateFormat df =
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", new DateFormatSymbols(Locale.US));
+ if (useUTC) {
+ df.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
String parsedDateString = df.format(parsedDate);
assertEquals(expected, parsedDateString, "failed to match: " + dateString);
}

original_message: TIKA-3815: fix RFC822ParserTest failing tests


message: Remove double this

diff:
@@ -153,7 +153,7 @@ queries. I will also discuss writing stand alone Java and Groovy applications
as well as using the Gremlin Console. I even slipped a couple of Ruby examples in
too!
-NOTE: In the first few sections of this this book I have mainly focussed on
+NOTE: In the first few sections of this book I have mainly focussed on
showing the different types of query that you can issue using Gremlin. I have not
tried to show all of the output that you will get back from entering these queries
but have selectively shown examples of output. I go a lot deeper into things in

repo: krlawrence/graph

original_message: Remove double this


message: Remove duplicate also

diff:
@@ -2777,7 +2777,7 @@ g.V().hasId(8).values('code')
DFW
----
-We could also have also written the above query as follows.
+We could also have written the above query as follows.
[source,groovy]
----

original_message: Remove duplicate also


message: Fix typo in fly

diff:
@@ -3636,7 +3636,7 @@ g.V().hasLabel('airport').order().by('code').limit(20).values('code').fold()
[AAE,AAL,AAN,AAQ,AAR,AAT,AAX,AAY,ABA,ABB,ABD,ABE,ABI,ABJ,ABL,ABM,ABQ,ABR,ABS,ABT]
----
-Here is a similar example to the previous two. We find all of the places you can flky
+Here is a similar example to the previous two. We find all of the places you can fly
to from Austin (AUS) and sort the results, as before using the airport's IATA code,
but this time we also include the ICAO code for each airport in the result set.

original_message: Fix typo in fly