I am following this tutorial to build a simple deep learning app for traffic sign recognition. link
I made an own model, and I also tried it with the model in this repository: link
When I run the app from xcode on my iPhone, it only recognizes traffic signs if they are rotated RIGHT by 90 degrees. How could I fix this?
Here is my AppDelegate.swift:
//
// AppDelegate.swift
// trafficsign
//
// Created by administrator on 2020. 11. 11..
// Copyright © 2020. administrator. All rights reserved.
//
import UIKit
@UIApplicationMain
class AppDelegate: UIResponder, UIApplicationDelegate {
var window: UIWindow?
func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
// Override point for customization after application launch.
// Override point for customization after application launch.
window = UIWindow()
window?.makeKeyAndVisible()
let vc = ViewController()
window?.rootViewController = vc
return true
}
}
My SceneDelegate.swift:
//
// SceneDelegate.swift
// trafficsign
//
// Created by administrator on 2020. 11. 11..
// Copyright © 2020. administrator. All rights reserved.
//
import UIKit
class SceneDelegate: UIResponder, UIWindowSceneDelegate {
var window: UIWindow?
func scene(_ scene: UIScene, willConnectTo session: UISceneSession, options connectionOptions: UIScene.ConnectionOptions) {
// Use this method to optionally configure and attach the UIWindow `window` to the provided UIWindowScene `scene`.
// If using a storyboard, the `window` property will automatically be initialized and attached to the scene.
// This delegate does not imply the connecting scene or session are new (see `application:configurationForConnectingSceneSession` instead).
guard let windowScene = (scene as? UIWindowScene) else { return }
window = UIWindow(windowScene: windowScene)
window?.rootViewController = ViewController()
window?.makeKeyAndVisible()
}
func sceneDidDisconnect(_ scene: UIScene) {
// Called as the scene is being released by the system.
// This occurs shortly after the scene enters the background, or when its session is discarded.
// Release any resources associated with this scene that can be re-created the next time the scene connects.
// The scene may re-connect later, as its session was not neccessarily discarded (see `application:didDiscardSceneSessions` instead).
}
func sceneDidBecomeActive(_ scene: UIScene) {
// Called when the scene has moved from an inactive state to an active state.
// Use this method to restart any tasks that were paused (or not yet started) when the scene was inactive.
}
func sceneWillResignActive(_ scene: UIScene) {
// Called when the scene will move from an active state to an inactive state.
// This may occur due to temporary interruptions (ex. an incoming phone call).
}
func sceneWillEnterForeground(_ scene: UIScene) {
// Called as the scene transitions from the background to the foreground.
// Use this method to undo the changes made on entering the background.
}
func sceneDidEnterBackground(_ scene: UIScene) {
// Called as the scene transitions from the foreground to the background.
// Use this method to save data, release shared resources, and store enough scene-specific state information
// to restore the scene back to its current state.
}
}
And most importantly my ViewController.swift:
//
// ViewController.swift
// trafficsign
//
// Created by administrator on 2020. 11. 11..
// Copyright © 2020. administrator. All rights reserved.
//
import UIKit
import AVFoundation
import Vision
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
let label: UILabel = {
let label = UILabel()
label.textColor = .white
label.translatesAutoresizingMaskIntoConstraints = false
label.text = "Label"
label.font = label.font.withSize(30)
return label
}()
override func viewDidLoad() {
super.viewDidLoad()
setupCaptureSession()
view.addSubview(label)
setupLabel()
}
override func didReceiveMemoryWarning() {
// call the parent function
super.didReceiveMemoryWarning()
// Dispose of any resources that can be recreated.
}
func setupCaptureSession() {
// create a new capture session
let captureSession = AVCaptureSession()
// find the available cameras
let availableDevices = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera], mediaType: AVMediaType.video, position: .back).devices
do {
// select a camera
if let captureDevice = availableDevices.first {
captureSession.addInput(try AVCaptureDeviceInput(device: captureDevice))
}
} catch {
// print an error if the camera is not available
print(error.localizedDescription)
}
// setup the video output to the screen and add output to our capture session
let captureOutput = AVCaptureVideoDataOutput()
captureSession.addOutput(captureOutput)
let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
previewLayer.frame = view.frame
view.layer.addSublayer(previewLayer)
// buffer the video and start the capture session
captureOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoQueue"))
captureSession.startRunning()
// // creates a new capture session
// let captureSession = AVCaptureSession()
//
// // search for available capture devices
// let availableDevices = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera], mediaType: AVMediaType.video, position: .back).devices
//
// // get capture device, add device input to capture session
// do {
// if let captureDevice = availableDevices.first {
// captureSession.addInput(try AVCaptureDeviceInput(device: captureDevice))
// }
// } catch {
// print(error.localizedDescription)
// }
//
// // setup output, add output to capture session
// let captureOutput = AVCaptureVideoDataOutput()
// captureSession.addOutput(captureOutput)
//
// captureOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoQueue"))
//
// let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
// previewLayer.frame = view.frame
// previewLayer.videoGravity = .resizeAspectFill
// view.layer.addSublayer(previewLayer)
//
// captureSession.startRunning()
}
// called everytime a frame is captured
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
// load our CoreML Pokedex model
guard let model = try? VNCoreMLModel(for: model_squeezeNet_TSR().model) else { return }
// run an inference with CoreML
let request = VNCoreMLRequest(model: model) { (finishedRequest, error) in
// grab the inference results
guard let results = finishedRequest.results as? [VNClassificationObservation] else { return }
// grab the highest confidence result
guard let Observation = results.first else { return }
// create the label text components
let predclass = "\(Observation.identifier)"
let predconfidence = String(format: "%.02f%", Observation.confidence * 100)
// set the label text
DispatchQueue.main.async(execute: {
self.label.text = "\(predclass) \(predconfidence)"
})
}
// create a Core Video pixel buffer which is an image buffer that holds pixels in main memory
// Applications generating frames, compressing or decompressing video, or using Core Image
// can all make use of Core Video pixel buffers
guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// execute the request
try? VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]).perform([request])
// guard let model = try? VNCoreMLModel(for: model_squeezeNet_TSR().model) else { return }
// let request = VNCoreMLRequest(model: model) { (finishedRequest, error) in
// guard let results = finishedRequest.results as? [VNClassificationObservation] else { return }
// guard let Observation = results.first else { return }
//
// DispatchQueue.main.async(execute: {
// self.label.text = "\(Observation.identifier)"
// print(Observation.confidence)
// })
// }
// guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// // executes request
// try? VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]).perform([request])
}
func setupLabel() {
label.centerXAnchor.constraint(equalTo: view.centerXAnchor).isActive = true
label.bottomAnchor.constraint(equalTo: view.bottomAnchor, constant: -50).isActive = true
}
}
Images that come from the camera are always natively in landscape orientation.
You can fix this by telling the video connection to use portrait orientation instead, or rotate the image yourself, but the easiest solution is to pass the orientation into the VNImageRequestHandler.
It's either .landscapeLeft or .landscapeRight, I always forget which is the correct one.
Also note that you should not be creating your VNCoreMLModel and VNCoreMLRequest instances every time you receive a frame from the video capture. Allocate them once and reuse them.