[FEATURE, CGL] added new "sandwich" engine

cli-httpd/main.go

@@ -7,7 +7,7 @@ import (
 	"net/http"
 
 	"github.com/couchbaselabs/logg"
-	"github.com/tleyden/open-ocr"
+	"github.com/xf0e/open-ocr"
 )
 
 // This assumes that there is a worker running
@@ -20,6 +20,7 @@ func init() {
 	logg.LogKeys["OCR_WORKER"] = true
 	logg.LogKeys["OCR_HTTP"] = true
 	logg.LogKeys["OCR_TESSERACT"] = true
+	logg.LogKeys["OCR_SANDWICH"] = true
 }
 
 func main() {

cli-preprocessor/main.go

@@ -18,6 +18,7 @@ func init() {
 	logg.LogKeys["PREPROCESSOR_WORKER"] = true
 	logg.LogKeys["OCR_HTTP"] = true
 	logg.LogKeys["OCR_TESSERACT"] = true
+	logg.LogKeys["OCR_SANDWICH"] = true
 }
 
 func main() {

cli-worker/main.go

@@ -2,9 +2,8 @@ package main
 
 import (
 	"fmt"
-
 	"github.com/couchbaselabs/logg"
-	"github.com/tleyden/open-ocr"
+	"github.com/xf0e/open-ocr"
 )
 
 // This assumes that there is a rabbit mq running
@@ -16,14 +15,15 @@ func init() {
 	logg.LogKeys["OCR_WORKER"] = true
 	logg.LogKeys["OCR_HTTP"] = true
 	logg.LogKeys["OCR_TESSERACT"] = true
+	logg.LogKeys["OCR_SANDWICH"] = true
 }
 
 func main() {
 
 	noOpFlagFunc := ocrworker.NoOpFlagFunction()
 	rabbitConfig := ocrworker.DefaultConfigFlagsOverride(noOpFlagFunc)
 
-	// inifinite loop, since sometimes worker <-> rabbitmq connection
+	// infinite loop, since sometimes worker <-> rabbitmq connection
 	// gets broken.  see https://github.com/tleyden/open-ocr/issues/4
 	for {
 		logg.LogTo("OCR_WORKER", "Creating new OCR Worker")

convert-pdf.go

@@ -63,7 +63,6 @@ func (c ConvertPdf) preprocess(ocrRequest *OcrRequest) error {
 	if err != nil {
 		logg.LogFatal("Error running command: %s. out: %s", err, out)
 	}
-	logg.LogTo("PREPROCESSOR_WORKER", "output: %v", string(out))
 
 	// read bytes from output file
 	resultBytes, err := ioutil.ReadFile(tmpFileNameOutput)

ocr_engine.go

@@ -10,9 +10,10 @@ import (
 type OcrEngineType int
 
 const (
-	ENGINE_TESSERACT = OcrEngineType(iota)
-	ENGINE_GO_TESSERACT
-	ENGINE_MOCK
+	EngineTesseract = OcrEngineType(iota)
+	EngineGoTesseract
+	EngineSandwichTesseract
+	EngineMock
 )
 
 type OcrEngine interface {
@@ -21,22 +22,26 @@ type OcrEngine interface {
 
 func NewOcrEngine(engineType OcrEngineType) OcrEngine {
 	switch engineType {
-	case ENGINE_MOCK:
+	case EngineMock:
 		return &MockEngine{}
-	case ENGINE_TESSERACT:
+	case EngineTesseract:
 		return &TesseractEngine{}
+	case EngineSandwichTesseract:
+		return &SandwichEngine{}
 	}
 	return nil
 }
 
 func (e OcrEngineType) String() string {
 	switch e {
-	case ENGINE_MOCK:
+	case EngineMock:
 		return "ENGINE_MOCK"
-	case ENGINE_TESSERACT:
+	case EngineTesseract:
 		return "ENGINE_TESSERACT"
-	case ENGINE_GO_TESSERACT:
+	case EngineGoTesseract:
 		return "ENGINE_GO_TESSERACT"
+	case EngineSandwichTesseract:
+		return "ENGINE_SANDWICH_TESSERACT"
 
 	}
 	return ""
@@ -50,14 +55,16 @@ func (e *OcrEngineType) UnmarshalJSON(b []byte) (err error) {
 		engineString := strings.ToUpper(engineTypeStr)
 		switch engineString {
 		case "TESSERACT":
-			*e = ENGINE_TESSERACT
+			*e = EngineTesseract
 		case "GO_TESSERACT":
-			*e = ENGINE_GO_TESSERACT
+			*e = EngineGoTesseract
+		case "SANDWICH":
+			*e = EngineSandwichTesseract
 		case "MOCK":
-			*e = ENGINE_MOCK
+			*e = EngineMock
 		default:
 			logg.LogWarn("Unexpected OcrEngineType json: %v", engineString)
-			*e = ENGINE_MOCK
+			*e = EngineMock
 		}
 		return nil
 	}

ocr_engine_test.go

@@ -17,7 +17,7 @@ func TestOcrEngineTypeJson(t *testing.T) {
 		logg.LogError(err)
 	}
 	assert.True(t, err == nil)
-	assert.Equals(t, ocrRequest.EngineType, ENGINE_TESSERACT)
+	assert.Equals(t, ocrRequest.EngineType, EngineTesseract)
 	logg.LogTo("TEST", "ocrRequest: %v", ocrRequest)
 
 }

ocr_http_handler.go

@@ -42,7 +42,7 @@ func (s *OcrHttpHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	logg.LogTo("OCR_HTTP", "ocrResult: %v", ocrResult)
+	// logg.LogTo("OCR_HTTP", "ocrResult: %v", ocrResult)
 
 	fmt.Fprintf(w, ocrResult.Text)
 

ocr_http_handler_test.go

@@ -34,7 +34,7 @@ func DisabledTestOcrHttpHandlerIntegration(t *testing.T) {
 
 	ocrRequest := OcrRequest{
 		ImgUrl:     "http://localhost:8081/img",
-		EngineType: ENGINE_MOCK,
+		EngineType: EngineMock,
 	}
 	jsonBytes, err := json.Marshal(ocrRequest)
 	if err != nil {

ocr_http_multipart_handler.go

@@ -111,7 +111,7 @@ func (s *OcrHttpMultipartHandler) ServeHTTP(w http.ResponseWriter, req *http.Req
 		return
 	}
 
-	logg.LogTo("OCR_HTTP", "ocrResult: %v", ocrResult)
+	// logg.LogTo("OCR_HTTP", "ocrResult: %v", ocrResult)
 
 	fmt.Fprintf(w, ocrResult.Text)
 

ocr_rpc_client.go

@@ -3,10 +3,10 @@ package ocrworker
 import (
 	"encoding/json"
 	"fmt"
-	"time"
-	"github.com/nu7hatch/gouuid"
 	"github.com/couchbaselabs/logg"
+	"github.com/nu7hatch/gouuid"
 	"github.com/streadway/amqp"
+	"time"
 )
 
 const (
@@ -54,11 +54,11 @@ func (c *OcrRpcClient) DecodeImage(ocrRequest OcrRequest) (OcrResult, error) {
 	if err := c.channel.ExchangeDeclare(
 		c.rabbitConfig.Exchange,     // name
 		c.rabbitConfig.ExchangeType, // type
-		true,  // durable
-		false, // auto-deleted
-		false, // internal
-		false, // noWait
-		nil,   // arguments
+		true,                        // durable
+		false,                       // auto-deleted
+		false,                       // internal
+		false,                       // noWait
+		nil,                         // arguments
 	); err != nil {
 		return OcrResult{}, err
 	}
@@ -167,8 +167,8 @@ func (c OcrRpcClient) subscribeCallbackQueue(correlationUuid string, rpcResponse
 		callbackQueue.Name,      // name of the queue
 		callbackQueue.Name,      // bindingKey
 		c.rabbitConfig.Exchange, // sourceExchange
-		false, // noWait
-		nil,   // arguments
+		false,                   // noWait
+		nil,                     // arguments
 	); err != nil {
 		return amqp.Queue{}, err
 	}
@@ -200,10 +200,10 @@ func (c OcrRpcClient) handleRpcResponse(deliveries <-chan amqp.Delivery, correla
 		if d.CorrelationId == correlationUuid {
 			logg.LogTo(
 				"OCR_CLIENT",
-				"got %dB delivery: [%v] %q.  Reply to: %v",
+				"got %dB delivery(first 64 Bytes): [%v] %q.  Reply to: %v",
 				len(d.Body),
 				d.DeliveryTag,
-				d.Body,
+				d.Body[0:64],
 				d.ReplyTo,
 			)
 

ocr_rpc_client_test.go

@@ -47,7 +47,7 @@ func DisabledTestOcrRpcClientIntegration(t *testing.T) {
 
 	for i := 0; i < 50; i++ {
 
-		ocrRequest := OcrRequest{ImgUrl: testImageUrl, EngineType: ENGINE_MOCK}
+		ocrRequest := OcrRequest{ImgUrl: testImageUrl, EngineType: EngineMock}
 		decodeResult, err := ocrClient.DecodeImage(ocrRequest)
 		if err != nil {
 			logg.LogTo("TEST", "err: %v", err)

ocr_rpc_worker.go

@@ -56,11 +56,11 @@ func (w OcrRpcWorker) Run() error {
 	if err = w.channel.ExchangeDeclare(
 		w.rabbitConfig.Exchange,     // name of the exchange
 		w.rabbitConfig.ExchangeType, // type
-		true,  // durable
-		false, // delete when complete
-		false, // internal
-		false, // noWait
-		nil,   // arguments
+		true,                        // durable
+		false,                       // delete when complete
+		false,                       // internal
+		false,                       // noWait
+		nil,                         // arguments
 	); err != nil {
 		return err
 	}
@@ -72,7 +72,7 @@ func (w OcrRpcWorker) Run() error {
 	queue, err := w.channel.QueueDeclare(
 		queueName, // name of the queue
 		true,      // durable
-		false,     // delete when usused
+		false,     // delete when unused
 		false,     // exclusive
 		false,     // noWait
 		nil,       // arguments
@@ -87,8 +87,8 @@ func (w OcrRpcWorker) Run() error {
 		queue.Name,                // name of the queue
 		w.rabbitConfig.RoutingKey, // bindingKey
 		w.rabbitConfig.Exchange,   // sourceExchange
-		false, // noWait
-		nil,   // arguments
+		false,                     // noWait
+		nil,                       // arguments
 	); err != nil {
 		return err
 	}
@@ -115,7 +115,7 @@ func (w OcrRpcWorker) Run() error {
 func (w *OcrRpcWorker) Shutdown() error {
 	// will close() the deliveries channel
 	if err := w.channel.Cancel(w.tag, true); err != nil {
-		return fmt.Errorf("Worker cancel failed: %s", err)
+		return fmt.Errorf("worker cancel failed: %s", err)
 	}
 
 	if err := w.conn.Close(); err != nil {
@@ -145,7 +145,7 @@ func (w *OcrRpcWorker) handle(deliveries <-chan amqp.Delivery, done chan error)
 			logg.LogError(fmt.Errorf(msg, err))
 		}
 
-		logg.LogTo("OCR_WORKER", "Sending rpc response: %v", ocrResult)
+		// logg.LogTo("OCR_WORKER", "Sending rpc response: %v", ocrResult)
 		err = w.sendRpcResponse(ocrResult, d.ReplyTo, d.CorrelationId)
 		if err != nil {
 			msg := "Error returning ocr result: %v.  Error: %v"
@@ -166,7 +166,7 @@ func (w *OcrRpcWorker) resultForDelivery(d amqp.Delivery) (OcrResult, error) {
 	ocrResult := OcrResult{Text: "Error"}
 	err := json.Unmarshal(d.Body, &ocrRequest)
 	if err != nil {
-		msg := "Error unmarshaling json: %v.  Error: %v"
+		msg := "Error unmarshalling json: %v.  Error: %v"
 		errMsg := fmt.Sprintf(msg, string(d.Body), err)
 		logg.LogError(fmt.Errorf(errMsg))
 		ocrResult.Text = errMsg
@@ -237,7 +237,7 @@ func confirmDeliveryWorker(ack, nack chan uint64) {
 	case tag := <-nack:
 		logg.LogTo("OCR_WORKER", "failed to confirm delivery: %v", tag)
 	case <-time.After(RPC_RESPONSE_TIMEOUT):
-		// this is bad, the worker will probably be dsyfunctional
+		// this is bad, the worker will probably be dysfunctional
 		// at this point, so panic
 		logg.LogPanic("timeout trying to confirm delivery")
 	}

ocr_util.go

@@ -1,10 +1,14 @@
 package ocrworker
 
 import (
+	"fmt"
+	"github.com/couchbaselabs/logg"
 	"io/ioutil"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strings"
 
 	"github.com/nu7hatch/gouuid"
 )
@@ -59,3 +63,54 @@ func createTempFileName() (string, error) {
 	uuidStr := uuidRaw.String()
 	return filepath.Join(tempDir, uuidStr), nil
 }
+
+func readFirstBytes(filePath string, nBytesToRead uint) ([]byte, error) {
+
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	buffer := make([]byte, nBytesToRead)
+	_, err = file.Read(buffer)
+	if err != nil {
+		return nil, err
+	}
+	return buffer, nil
+}
+
+// detect uploaded file type
+func detectFileType(buffer []byte) string {
+	logg.LogTo("OCR_SANDWICH", "OK, this is buffer: %v", buffer)
+	fileType := ""
+	if len(buffer) > 3 &&
+		buffer[0] == 0x25 && buffer[1] == 0x50 &&
+		buffer[2] == 0x44 && buffer[3] == 0x46 {
+		fileType = strings.ToUpper("PDF")
+	} else if len(buffer) > 3 &&
+		((buffer[0] == 0x49 && buffer[1] == 0x49 && buffer[2] == 0x2A && buffer[3] == 0x0) ||
+			(buffer[0] == 0x4D && buffer[1] == 0x4D && buffer[2] == 0x0 && buffer[3] == 0x2A)) {
+		fileType = strings.ToUpper("TIFF")
+	} else {
+		fileType = strings.ToUpper("UNKNOWN")
+	}
+	return fileType
+}
+
+// if sandwich engine gets a TIFF image instead of PDF file
+// we need to convert the input file to pdf first since pdfsandwich can't handle images
+func convertImageToPdf(inputFilename string) string {
+	logg.LogTo("OCR_SANDWICH", "got image file instead of pdf, trying to convert it...")
+
+	tmpFileImgToPdf := fmt.Sprintf("%s%s", inputFilename, ".pdf")
+	cmd := exec.Command("convert", inputFilename, tmpFileImgToPdf)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		logg.LogWarn("OCR_SANDWICH", "error exec convert for transforming TIFF to PDF: %v %v", err, string(output))
+		return ""
+	}
+
+	return tmpFileImgToPdf
+
+}