iOS: keep track of the keyboard, and scroll the next search result into view.…

iOS: keep track of the keyboard, and scroll the next search result into view. Reimplement RenderCache (+2 squashed commits)
Squashed commits:
[3c3f36f] iOS: quieten warnings
[8eae946] iOS: display search results in an overlay view

Change-Id: I04a38943d5a22b8e6a52ae854e65f01bf43fda7b
Reviewed-on: https://gerrit.libreoffice.org/48100Reviewed-by: jan iversen <jani@libreoffice.org>
Tested-by: jan iversen <jani@libreoffice.org>

ios/LibreOfficeLight/LibreOfficeLight.xcodeproj/project.pbxproj

@@ -33,6 +33,7 @@
 		39B091CE1E5F0BB800682A59 /* unorc in Resources */ = {isa = PBXBuildFile; fileRef = 39B08B9C1E5F0BB600682A59 /* unorc */; };
 		39E950531FC9842000D82C49 /* source in Resources */ = {isa = PBXBuildFile; fileRef = 39E950521FC9842000D82C49 /* source */; };
 		39EF4E2F1FA500C9001914AC /* PropertiesController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 39EF4E2E1FA500C9001914AC /* PropertiesController.swift */; };
+		FCAB1CB82009DB6900F1CC34 /* DocumentOverlaysView.swift in Sources */ = {isa = PBXBuildFile; fileRef = FCAB1CB72009DB6900F1CC34 /* DocumentOverlaysView.swift */; };
 		FCC2E3FA2004A01500CEB504 /* Document.swift in Sources */ = {isa = PBXBuildFile; fileRef = FCC2E3F62004A01400CEB504 /* Document.swift */; };
 		FCC2E3FC2004A01500CEB504 /* LibreOfficeKitWrapper.swift in Sources */ = {isa = PBXBuildFile; fileRef = FCC2E3F82004A01400CEB504 /* LibreOfficeKitWrapper.swift */; };
 		FCC2E3FD2004A01500CEB504 /* LOKitThread.swift in Sources */ = {isa = PBXBuildFile; fileRef = FCC2E3F92004A01400CEB504 /* LOKitThread.swift */; };
@@ -76,6 +77,7 @@
 		39E950521FC9842000D82C49 /* source */ = {isa = PBXFileReference; lastKnownFileType = folder; name = source; path = ../source; sourceTree = "<group>"; };
 		39EE81531FA644E800B73AB8 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
 		39EF4E2E1FA500C9001914AC /* PropertiesController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PropertiesController.swift; sourceTree = "<group>"; };
+		FCAB1CB72009DB6900F1CC34 /* DocumentOverlaysView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DocumentOverlaysView.swift; sourceTree = "<group>"; };
 		FCC2E3F62004A01400CEB504 /* Document.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Document.swift; sourceTree = "<group>"; };
 		FCC2E3F82004A01400CEB504 /* LibreOfficeKitWrapper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LibreOfficeKitWrapper.swift; sourceTree = "<group>"; };
 		FCC2E3F92004A01400CEB504 /* LOKitThread.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LOKitThread.swift; sourceTree = "<group>"; };
@@ -161,6 +163,7 @@
 				39503A6F1F94C4AC00F19C78 /* lokit-Bridging-Header.h */,
 				397E08FD1E597BD8001374E0 /* AppDelegate.swift */,
 				3992D8591E5B762A00BEA987 /* DocumentController.swift */,
+				FCAB1CB72009DB6900F1CC34 /* DocumentOverlaysView.swift */,
 				FCC2E3FE2004B59B00CEB504 /* DocumentTiledView.swift */,
 				39284DB21FA5F207006F43E4 /* DocumentActions.swift */,
 				39EF4E2E1FA500C9001914AC /* PropertiesController.swift */,
@@ -303,6 +306,7 @@
 			files = (
 				FCC2E4032004B72700CEB504 /* Util.swift in Sources */,
 				392ED9B31E5E4B03005C8435 /* ViewPrintManager.swift in Sources */,
+				FCAB1CB82009DB6900F1CC34 /* DocumentOverlaysView.swift in Sources */,
 				399648471E5B87DC00E73E83 /* ViewProperties.swift in Sources */,
 				FCC2E3FC2004A01500CEB504 /* LibreOfficeKitWrapper.swift in Sources */,
 				39284DB31FA5F207006F43E4 /* DocumentActions.swift in Sources */,

ios/LibreOfficeLight/LibreOfficeLight/DocumentController.swift

@@ -17,6 +17,7 @@ class DocumentController: UIViewController, MenuDelegate, UIDocumentBrowserViewC
     var document: DocumentHolder? = nil
 
     var documentView: DocumentTiledView? = nil
+    var documentOverlaysView: DocumentOverlaysView? = nil
 
     // *** Handling of DocumentController
     // this is normal functions every controller must implement
@@ -30,6 +31,11 @@ class DocumentController: UIViewController, MenuDelegate, UIDocumentBrowserViewC
     @IBOutlet weak var progressBar: UIProgressView!
     @IBOutlet weak var searchBar: UISearchBar!
 
+    deinit
+    {
+        NotificationCenter.default.removeObserver(self)
+    }
+
     // called once controller is loaded
     override func viewDidLoad()
     {
@@ -46,8 +52,15 @@ class DocumentController: UIViewController, MenuDelegate, UIDocumentBrowserViewC
         LOKitThread.instance.progressDelegate = self
     }
 
+    override func viewWillAppear(_ animated: Bool)
+    {
+        super.viewWillAppear(animated)
+        registerKeyboardNotifications()
+    }
+
     override func viewDidAppear(_ animated: Bool)
     {
+        super.viewDidAppear(animated)
         let res = Bundle.main.url(forResource: "example", withExtension: "odt")
         //let res = Bundle.main.url(forResource: "example2", withExtension: "docx")
 
@@ -370,7 +383,7 @@ class DocumentController: UIViewController, MenuDelegate, UIDocumentBrowserViewC
     /// Sets the document to use and set's up it's view. Should be called on the main thread
     public func setDocument(doc: DocumentHolder)
     {
-        if let existingDoc = self.document
+        if let _ = self.document
         {
             // TODO - cleanup
             self.document = nil
@@ -380,9 +393,13 @@ class DocumentController: UIViewController, MenuDelegate, UIDocumentBrowserViewC
             exisitingView.removeFromSuperview()
             self.documentView = nil // forces the close of the view and it's held documents before we setup the new one
         }
+        // also remove current overlays and start fresh
+        documentOverlaysView?.removeFromSuperview()
 
         // setup the new doc view
         self.document = doc
+        // setup delegates
+        doc.searchDelegate = self
 
         let frameToUse = self.scrollView.frame
 
@@ -392,6 +409,11 @@ class DocumentController: UIViewController, MenuDelegate, UIDocumentBrowserViewC
         self.scrollView.contentSize = docView.frame.size
         self.documentView = docView
 
+        // overlay view
+        let overlay = DocumentOverlaysView(docTiledView: docView)
+        docView.addSubview(overlay)
+        self.documentOverlaysView = overlay
+
         // debugging view borders
         /*
         self.scrollView.layer.borderColor = UIColor.red.cgColor
@@ -493,3 +515,53 @@ extension DocumentController: UISearchBarDelegate
     }
 }
 
+extension DocumentController: SearchDelegate
+{
+    func searchNotFound()
+    {
+        // TODO: tell user somehow
+        self.documentOverlaysView?.clearSearchResults()
+    }
+
+    func searchResultSelection(searchResults: SearchResults)
+    {
+        self.documentOverlaysView?.setSearchResults(searchResults: searchResults)
+    }
+}
+
+/// Keyboard notifications
+extension DocumentController
+{
+
+    func registerKeyboardNotifications()
+    {
+        NotificationCenter.default.addObserver(self,
+                                               selector: #selector(keyboardWillShow(notification:)),
+                                               name: NSNotification.Name.UIKeyboardWillShow,
+                                               object: nil)
+        NotificationCenter.default.addObserver(self,
+                                               selector: #selector(keyboardWillHide(notification:)),
+                                               name: NSNotification.Name.UIKeyboardWillHide,
+                                               object: nil)
+    }
+
+    @objc func keyboardWillShow(notification: NSNotification)
+    {
+
+        let userInfo: NSDictionary = notification.userInfo! as NSDictionary
+        guard let keyboardInfo = userInfo[UIKeyboardFrameEndUserInfoKey] as? NSValue else { return }
+        print(userInfo)
+        let keyboardSize = keyboardInfo.cgRectValue.size
+        print("keyboardWillShow \(keyboardSize)")
+        let contentInsets = UIEdgeInsets(top: 0, left: 0, bottom: keyboardSize.height, right: 0)
+        scrollView.contentInset = contentInsets
+        scrollView.scrollIndicatorInsets = contentInsets
+    }
+
+    @objc func keyboardWillHide(notification: NSNotification)
+    {
+        print("keyboardWillHide")
+        scrollView.contentInset = .zero
+        scrollView.scrollIndicatorInsets = .zero
+    }
+}

ios/LibreOfficeLight/LibreOfficeLight/DocumentOverlaysView.swift

+//
+// This file is part of the LibreOffice project.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+
+import UIKit
+
+public class DocumentOverlaysView: UIView
+{
+    var searchSubViews: [UIView] = []
+    weak var documentTiledView: DocumentTiledView? = nil
+
+    public init(docTiledView: DocumentTiledView)
+    {
+        self.documentTiledView = docTiledView
+        super.init(frame: docTiledView.frame)
+
+        self.layer.compositingFilter = "multiplyBlendMode"
+    }
+
+    required public init?(coder aDecoder: NSCoder)
+    {
+        fatalError("init(coder:) has not been implemented")
+    }
+
+    public func clearSearchResults()
+    {
+        for v in self.searchSubViews
+        {
+            v.removeFromSuperview()
+        }
+        searchSubViews = []
+    }
+
+    public func setSearchResults(searchResults: SearchResults)
+    {
+        clearSearchResults()
+
+        guard let documentTiledView = self.documentTiledView else { return }
+
+        if let srs = searchResults.searchResultSelection
+        {
+            let allTheRects = srs.flatMap { $0.rectsAsCGRects }
+                .flatMap { $0 }
+                .map { documentTiledView.twipsToPixels(rect: $0) }
+
+            for rect in allTheRects
+            {
+                let subView = UIView(frame: rect)
+                subView.backgroundColor = UIColor.yellow // TODO
+                subView.layer.compositingFilter = "multiplyBlendMode"
+                self.addSubview(subView)
+                searchSubViews.append(subView)
+            }
+
+            if let first = allTheRects.first
+            {
+                if let scrollView = self.superview?.superview as? UIScrollView
+                {
+                    scrollView.scrollRectToVisible(first, animated: true)
+                }
+            }
+        }
+    }
+}

ios/LibreOfficeLight/LibreOfficeLight/DocumentTiledView.swift

@@ -18,24 +18,10 @@ class DocumentTiledLayer : CATiledLayer
     }
 }
 
-open class CachedRender
-{
-    open let x: CGFloat
-    open let y: CGFloat
-    open let scale: CGFloat
-    open let image: CGImage
 
-    public init(x: CGFloat, y: CGFloat, scale: CGFloat, image: CGImage)
-    {
-        self.x = x
-        self.y = y
-        self.scale = scale
-        self.image = image
-    }
-}
 
 
-class DocumentTiledView: UIView
+public class DocumentTiledView: UIView
 {
     var myScale: CGFloat
 
@@ -47,7 +33,7 @@ class DocumentTiledView: UIView
 
     var drawCount = 0
 
-    let drawLock = NSLock()
+
 
     // Create a new view with the desired frame and scale.
     public init(frame: CGRect, document: DocumentHolder, scale: CGFloat)
@@ -89,20 +75,29 @@ class DocumentTiledView: UIView
 
     }
 
-    required init?(coder aDecoder: NSCoder)
+    required public init?(coder aDecoder: NSCoder)
     {
         fatalError("init(coder:) has not been implemented")
     }
 
+    public func twipsToPixels(rect: CGRect) -> CGRect
+    {
+        return rect.applying(CGAffineTransform(scaleX: 1.0/initialScaleFactor, y: 1.0/initialScaleFactor ))
+    }
 
+    public func pixelsToTwips(rect: CGRect) -> CGRect
+    {
+        return rect.applying(CGAffineTransform(scaleX: initialScaleFactor, y: initialScaleFactor ))
+    }
 
-    override class var layerClass : AnyClass
+
+    override public class var layerClass : AnyClass
     {
         return DocumentTiledLayer.self
     }
 
 
-    override func draw(_ r: CGRect)
+    override public func draw(_ r: CGRect)
     {
         // UIView uses the existence of -drawRect: to determine if it should allow its CALayer
         // to be invalidated, which would then lead to the layer creating a backing store and
@@ -112,7 +107,7 @@ class DocumentTiledView: UIView
     }
 
     // Draw the CGPDFPageRef into the layer at the correct scale.
-    override func draw(_ layer: CALayer, in context: CGContext)
+    override public func draw(_ layer: CALayer, in context: CGContext)
     {
 //        if self.superview == nil
 //        {
@@ -132,9 +127,6 @@ class DocumentTiledView: UIView
         let box: CGRect = context.boundingBoxOfClipPath
         let ctm: CGAffineTransform = context.ctm
 
-        drawLock.lock()
-        defer { drawLock.unlock() }
-
         drawCount += 1
         let filename = "tile\(drawCount).png"
 
@@ -150,7 +142,7 @@ class DocumentTiledView: UIView
 
         // This is where the magic happens
 
-        let pageRect = box.applying(CGAffineTransform(scaleX: initialScaleFactor, y: initialScaleFactor ))
+        let pageRect = pixelsToTwips(rect: box)
         print("  pageRect: \(pageRect.desc)")
 
         // Figure out how many pixels we need for the dimensions of our tile
@@ -164,9 +156,8 @@ class DocumentTiledView: UIView
 
         // we have to do the call synchronously, as the tile has to be painted now, on the current thread
         // TODO - cache the image, and check the cache before we do the sync call
-        let image = document.sync {
-            $0.paintTileToImage(canvasSize: canvasSize, tileRect: pageRect)
-        }
+        let image = document.paintTileToImage(canvasSize: canvasSize, tileRect: pageRect)
+
 
         if let img = image
         {
@@ -192,23 +183,6 @@ class DocumentTiledView: UIView
     }
 
 
-
-    /*
-    fileprivate func emptyCache()
-    {
-        cachedRenders.removeAll()
-    }
-
-    fileprivate func pruneCache()
-    {
-        let max = hasReceivedMemoryWarning ? CACHE_LOWMEM : CACHE_NORMAL
-        while cachedRenders.count > max
-        {
-            cachedRenders.popFirst()
-        }
-    }
- */
-
     deinit
     {
         self.document = nil

ios/LibreOfficeLight/LibreOfficeLight/LOKit/Document.swift

@@ -228,6 +228,7 @@ open class Document
      * @param pCallback the callback to invoke
      * @param pData the user data, will be passed to the callback on invocation
      */
+    @discardableResult
     public func registerCallback( callback: @escaping LibreOfficeCallback ) -> Int
     {
         let ret = Callbacks.register(callback: callback)
@@ -570,15 +571,9 @@ public extension Document
     public func paintTileToImage(canvasSize: CGSize,
                                  tileRect: CGRect) -> UIImage?
     {
-        // the scaling etc here is all black magic.
-        // I don't really understand whats going on, other than that this combination works...
 
         UIGraphicsBeginImageContextWithOptions(canvasSize, false, 1.0)
-        let ctx = UIGraphicsGetCurrentContext()!
-
-        //        print(ctx)
-        //        print(ctx.ctm)
-        //        print(ctx.userSpaceToDeviceSpaceTransform)
+        let _ = UIGraphicsGetCurrentContext()!
 
         self.paintTileToCurrentContext(canvasSize: canvasSize, tileRect: tileRect)
         let image = UIGraphicsGetImageFromCurrentImageContext()

ios/LibreOfficeLight/LibreOfficeLight/LOKit/LOKitThread.swift

@@ -119,6 +119,71 @@ public class LOKitThread
     }
 }
 
+
+open class CachedRender
+{
+    open let canvasSize: CGSize
+    open let tileRect: CGRect
+    open let image: UIImage
+
+    public init(canvasSize: CGSize, tileRect: CGRect, image: UIImage)
+    {
+        self.canvasSize = canvasSize
+        self.tileRect = tileRect
+        self.image = image
+    }
+}
+
+class RenderCache
+{
+    let CACHE_LOWMEM = 4
+    let CACHE_NORMAL = 20
+
+    var cachedRenders: [CachedRender] = []
+    var hasReceivedMemoryWarning = false
+
+    let lock = NSRecursiveLock()
+
+    func emptyCache()
+    {
+        lock.lock(); defer { lock.unlock() }
+
+        cachedRenders.removeAll()
+
+    }
+
+    func pruneCache()
+    {
+        lock.lock(); defer { lock.unlock() }
+
+        let max = hasReceivedMemoryWarning ? CACHE_LOWMEM : CACHE_NORMAL
+        while cachedRenders.count > max
+        {
+            cachedRenders.remove(at: 0)
+        }
+    }
+
+    func add(cachedRender: CachedRender)
+    {
+        lock.lock(); defer { lock.unlock() }
+
+        cachedRenders.append(cachedRender)
+        pruneCache()
+    }
+
+    func get(canvasSize: CGSize, tileRect: CGRect) -> UIImage?
+    {
+        lock.lock(); defer { lock.unlock() }
+
+        if let cr = cachedRenders.first(where: { $0.canvasSize == canvasSize && $0.tileRect == tileRect })
+        {
+            return cr.image
+        }
+        return nil
+    }
+
+}
+
 /**
  * Holds the document object so to enforce access in a thread safe way.
  */
@@ -127,6 +192,9 @@ public class DocumentHolder
     private let doc: Document
 
     public weak var delegate: DocumentUIDelegate? = nil
+    public weak var searchDelegate: SearchDelegate? = nil
+
+    private let cache = RenderCache()
 
     init(doc: Document)
     {
@@ -156,6 +224,27 @@ public class DocumentHolder
         }
     }
 
+    /// Paints a tile and return synchronously, using a cached version if it can
+    public func paintTileToImage(canvasSize: CGSize,
+                                 tileRect: CGRect) -> UIImage?
+    {
+        if let cached = cache.get(canvasSize: canvasSize, tileRect: tileRect)
+        {
+            return cached
+        }
+
+        let img = sync {
+            $0.paintTileToImage(canvasSize: canvasSize, tileRect: tileRect)
+        }
+        if let image = img
+        {
+            cache.add(cachedRender: CachedRender(canvasSize: canvasSize, tileRect: tileRect, image: image))
+        }
+
+        return img
+    }
+
+
     private func onDocumentEvent(type: LibreOfficeKitCallbackType, payload: String?)
     {
         print("onDocumentEvent type:\(type) payload:\(payload ?? "")")
@@ -182,20 +271,61 @@ public class DocumentHolder
             runOnMain {
                 self.delegate?.textSelectionEnd( rects: decodeRects(payload) )
             }
+
+        case LOK_CALLBACK_SEARCH_NOT_FOUND:
+            runOnMain {
+                self.searchDelegate?.searchNotFound()
+            }
+        case LOK_CALLBACK_SEARCH_RESULT_SELECTION:
+            runOnMain {
+                self.searchResults(payload: payload)
+            }
+
+
         default:
             print("onDocumentEvent type:\(type) not handled!")
         }
     }
 
+    private func searchResults(payload: String?)
+    {
+        if let d = payload, let data = d.data(using: .utf8)
+        {
+            let decoder = JSONDecoder()
+
+            do
+            {
+                let searchResults = try decoder.decode(SearchResults.self, from: data )
+
+                /*
+                if let srs = searchResults.searchResultSelection
+                {
+                    for par in srs
+                    {
+                        print("\(par.rectsAsCGRects)")
+                    }
+                }
+                */
+
+                self.searchDelegate?.searchResultSelection(searchResults: searchResults)
+            }
+            catch
+            {
+                print("Error decoding payload: \(error)")
+            }
+
+        }
+    }
+
     public func search(searchString: String, forwardDirection: Bool = true, from: CGPoint)
     {
         var rootJson = JSONObject()
 
         addProperty(&rootJson, "SearchItem.SearchString", "string", searchString);
-        addProperty(&rootJson, "SearchItem.Backward", "boolean", String(forwardDirection) );
+        addProperty(&rootJson, "SearchItem.Backward", "boolean", String(!forwardDirection) );
         addProperty(&rootJson, "SearchItem.SearchStartPointX", "long", String(describing: from.x) );
         addProperty(&rootJson, "SearchItem.SearchStartPointY", "long", String(describing: from.y) );
-        addProperty(&rootJson, "SearchItem.Command", "long", "1") // String.valueOf(0)); // search all == 1
+        addProperty(&rootJson, "SearchItem.Command", "long", "0") // String.valueOf(0)); // search all == 1
 
         if let jsonStr = encode(json: rootJson)
         {
@@ -240,7 +370,7 @@ public func decodeRects(_ payload: String?) -> [CGRect]?
     var ret = [CGRect]()
     for rectStr in pl.split(separator: ";")
     {
-        let coords = rectStr.split(separator: ",").flatMap { Double($0) }
+        let coords = rectStr.split(separator: ",").flatMap { Double($0.trimmingCharacters(in: .whitespacesAndNewlines)) }
         if coords.count == 4
         {
             let rect = CGRect(x: coords[0],
@@ -281,7 +411,69 @@ public protocol DocumentUIDelegate: class
     func textSelection(rects: [CGRect]? )
     func textSelectionStart(rects: [CGRect]? )
     func textSelectionEnd(rects: [CGRect]? )
+}
+
+public protocol SearchDelegate: class
+{
+    func searchNotFound()
+
+    func searchResultSelection(searchResults: SearchResults)
+}
 
+/**
+ Encodes this example json:
+ {
+ "searchString": "Office",
+ "highlightAll": "true",
+ "searchResultSelection": [
+ {
+ "part": "0",
+ "rectangles": "1951, 10743, 627, 239"
+ },
+ {
+ "part": "0",
+ "rectangles": "5343, 9496, 627, 287"
+ },
+ {
+ "part": "0",
+ "rectangles": "1951, 9256, 627, 239"
+ },
+ {
+ "part": "0",
+ "rectangles": "6502, 5946, 626, 287"
+ },
+ {
+ "part": "0",
+ "rectangles": "6686, 5658, 627, 287"
+ },
+ {
+ "part": "0",
+ "rectangles": "4103, 5418, 573, 239"
+ },
+ {
+ "part": "0",
+ "rectangles": "1951, 5418, 627, 239"
+ },
+ {
+ "part": "0",
+ "rectangles": "4934, 1658, 1586, 559"
+ }
+ ]
+ }
+*/
+public struct SearchResults: Codable
+{
+    public var searchString: String?
+    public var highlightAll: String?
+    public var searchResultSelection: Array<PartAndRectangles>?
+}
 
+public struct PartAndRectangles: Codable
+{
+    public var part: String?
+    public var rectangles: String?
 
+    public var rectsAsCGRects: [CGRect]? {
+        return decodeRects(self.rectangles)
+    }
 }

ios/LibreOfficeLight/LibreOfficeLight/LOKit/LibreOfficeKitWrapper.swift

@@ -141,6 +141,7 @@ open class LibreOffice
      * @param pCallback the callback to invoke
      * @param pData the user data, will be passed to the callback on invocation
      */
+    @discardableResult
     public func registerCallback( callback: @escaping LibreOfficeCallback ) -> Int
     {
         let ret = Callbacks.register(callback: callback)