外カメラ x スマホVR = デジタルメガネ
JSで手軽なWebMeganeもオススメながら、iPhoneネイティブアプリとして作ると、顔認識や、QRコード認識などが高速で楽しい！
リアルタイムにQRコード内容をデコードして、視界をオーバーレイ（上書き）するメガネができた。
「megane - qrdetector on GitHub」 Swift4で実装
ウィンクシャッターが便利だったGoogle Glassのように、手軽に写真撮影もついでに実装。
ボリュームダウンキーで、シャッター（初回のみダイアログがでるので、VRにする前に押しておこう）
ボリュームアップキーで、視界をズームすることもできる（現在は、1920x1080の画像をデジタルズーム）
こちら「オタマートで売ってるVRゴーグル」の一番安いもの。ボリュームアップダウンが上からアクセスできる！
ボリュームアップダウンキーのついたイヤホンを接続すると、手元でも操作できるようになって便利。
デジタルメガネなUXづくり、いろいろ試していきましょう。
鯖江の道の駅、西山公園前のバス停。日本語表記しかない標識にQRコードをつけておけば、リアルタイムに見ている人の母国語に変換できそう。
Let's hack your sight!
AppDelegate.swift
import UIKit
@UIApplicationMain
class AppDelegate: UIResponder, UIApplicationDelegate {
var window: UIWindow?
func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplicationLaunchOptionsKey: Any]?) -> Bool {
return true
}
func applicationWillResignActive(_ application: UIApplication) {
}
func applicationDidEnterBackground(_ application: UIApplication) {
NotificationCenter.default.post(name: NSNotification.Name(rawValue: "applicationDidEnterBackground"), object: nil)
}
func applicationWillEnterForeground(_ application: UIApplication) {
NotificationCenter.default.post(name: NSNotification.Name(rawValue: "applicationWillEnterForeground"), object: nil)
}
func applicationDidBecomeActive(_ application: UIApplication) {
}
func applicationWillTerminate(_ application: UIApplication) {
}
}
ViewController.swift
//
// ViewController.swift
// megane, QR code detector glass with VR goggle
//
// CC BY taisukef on 2018/05/24.
// https://fukuno.jig.jp/2133
//
import UIKit
import AVFoundation
import MediaPlayer
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
override func viewDidLoad() {
super.viewDidLoad()
self.view.backgroundColor = UIColor.black
self.imageView1 = UIImageView()
self.imageView2 = UIImageView()
let w = self.view.frame.width
let h = self.view.frame.height
let w2 = w / 2
let h2 = w2 * 1080 / 1920
let y = (h - h2) / 2
self.imageView1.frame = CGRect(x:0, y:y, width:w2, height:h2)
self.imageView2.frame = CGRect(x:self.view.frame.width / 2, y:y, width:w2, height:h2)
self.view.addSubview(self.imageView1)
self.view.addSubview(self.imageView2)
self.initNotificationsFromAppDelegate()
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
// Dispose of any resources that can be recreated.
}
var input:AVCaptureDeviceInput!
var output:AVCaptureVideoDataOutput!
var session:AVCaptureSession!
var camera:AVCaptureDevice!
var imageView1:UIImageView!
var imageView2:UIImageView!
override func viewWillAppear(_ animated: Bool) {
self.configureCamera()
self.listenVolumeButton()
}
// notifications foreground and background
func initNotificationsFromAppDelegate() {
NotificationCenter.default.addObserver(self, selector: #selector(type(of: self).viewWillEnterForeground(notification:)), name: NSNotification.Name(rawValue: "applicationWillEnterForeground"), object: nil)
NotificationCenter.default.addObserver(self, selector: #selector(type(of: self).viewDidEnterBackground(notification:)), name: NSNotification.Name(rawValue: "applicationDidEnterBackground"), object: nil)
}
@objc func viewWillEnterForeground(notification: NSNotification?) {
print("foreground")
self.listenVolumeButton()
}
@objc func viewDidEnterBackground(notification: NSNotification?) {
print("background")
self.removeVolumeButton()
}
//
override func viewDidDisappear(_ animated: Bool) {
session.stopRunning()
for output in session.outputs {
session.removeOutput(output)
}
for input in session.inputs {
session.removeInput(input)
}
session = nil
camera = nil
}
let DETECT_QRCODE = false
func configureCamera() {
session = AVCaptureSession()
// iPhone Xで実験
//session.sessionPreset = AVCaptureSession.Preset.cif352x288 // 34% 荒い
//session.sessionPreset = AVCaptureSession.Preset.vga640x480 // 47% 4:3 なかなかきれい
//session.sessionPreset = AVCaptureSession.Preset.iFrame1280x720 // CPU50% 16:9 かわらない？
//session.sessionPreset = AVCaptureSession.Preset.hd1280x720 // CPU50% 16:9 きれい
session.sessionPreset = AVCaptureSession.Preset.hd1920x1080 // CPU88% 16:9 かわらない？ iPhone6でもQRcode offならOK!
//session.sessionPreset = AVCaptureSession.Preset.hd4K3840x2160 // CPU93% 16:9 かわらない？ QRcode offなら実用的
camera = AVCaptureDevice.default(
AVCaptureDevice.DeviceType.builtInWideAngleCamera,
for: AVMediaType.video,
position: .back) // position: .front
do {
input = try AVCaptureDeviceInput(device: camera)
} catch let error as NSError {
print(error)
}
if (session.canAddInput(input)) {
session.addInput(input)
}
output = AVCaptureVideoDataOutput() // AVCapturePhotoOutput() 写真用
output?.videoSettings = [kCVPixelBufferPixelFormatTypeKey as AnyHashable : Int(kCVPixelFormatType_32BGRA)] as! [String : Any]
let queue:DispatchQueue = DispatchQueue(label: "myqueue", attributes: .concurrent)
output.setSampleBufferDelegate(self, queue: queue)
output.alwaysDiscardsLateVideoFrames = true // 間に合わないものは処理しない
if (session.canAddOutput(output)) {
session.addOutput(output)
}
session.startRunning()
}
var zoom:CGFloat = 1.0
func captureOutput(_: AVCaptureOutput, didOutput: CMSampleBuffer, from: AVCaptureConnection) {
// from.videoOrientation = .portrait //デバイスの向きを設定、縦の時
from.videoOrientation = .landscapeLeft //デバイスの向きを設定、landscape left の時
DispatchQueue.main.sync(execute: {
var image = self.imageFromSampleBuffer(sampleBuffer: didOutput)
image = resizeImage(image: image, ratio: zoom)
if DETECT_QRCODE {
image = drawQR(image: image)
}
self.imageView1.image = image
self.imageView2.image = image
})
}
func resizeImage(image: UIImage, ratio: CGFloat) -> UIImage {
if ratio == 1.0 {
return image
}
let iw = image.size.width / ratio
let ih = image.size.height / ratio
let size = CGSize(width: iw, height: ih)
UIGraphicsBeginImageContext(size)
image.draw(in: CGRect(origin: CGPoint(x:-(image.size.width - iw) / 2, y:-(image.size.height - ih) / 2), size: image.size))
let resimage = UIGraphicsGetImageFromCurrentImageContext()!
UIGraphicsEndImageContext()
return resimage
}
func drawQR(image: UIImage) -> UIImage {
UIGraphicsBeginImageContext(image.size)
let rect = CGRect(x:0, y:0, width:image.size.width, height:image.size.height)
image.draw(in: rect)
let g = UIGraphicsGetCurrentContext()!
g.setStrokeColor(UIColor.white.cgColor)
g.setLineWidth(1)
let font = UIFont.boldSystemFont(ofSize: 14)
let textStyle = NSMutableParagraphStyle.default.mutableCopy() as! NSMutableParagraphStyle
let textFontAttributes = [
NSAttributedStringKey.font: font,
NSAttributedStringKey.foregroundColor: UIColor.black,
NSAttributedStringKey.paragraphStyle: textStyle
]
// 顔認識もおもしろい
// let detector : CIDetector = CIDetector(ofType: CIDetectorTypeFace, context: nil, options:[CIDetectorAccuracy: CIDetectorAccuracyLow] )!
// 読める四角は今のところひとつだけ
// let detector : CIDetector = CIDetector(ofType: CIDetectorTypeRectangle, context: nil, options:[CIDetectorAccuracy: CIDetectorAccuracyHigh, CIDetectorAspectRatio: 1.0] )!
let detector : CIDetector = CIDetector(ofType: CIDetectorTypeQRCode, context: nil, options:[CIDetectorAccuracy: CIDetectorAccuracyHigh] )!
let features : NSArray = detector.features(in: CIImage(image: image)!) as NSArray
if features.count > 0 {
for feature in features as! [CIQRCodeFeature] {
var rect: CGRect = (feature as AnyObject).bounds
rect.origin.y = image.size.height - rect.origin.y - rect.size.height
// QRコードを上書き！
g.beginPath()
g.setFillColor(UIColor.white.cgColor)
g.addRect(rect)
g.fillPath()
feature.messageString?.draw(in: rect, withAttributes: textFontAttributes)
}
}
let resimage = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
return resimage!
}
func imageFromSampleBuffer(sampleBuffer: CMSampleBuffer) -> UIImage {
let imageBuffer: CVImageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)!
CVPixelBufferLockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0))
let baseAddress = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0)
let bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer)
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
let colorSpace = CGColorSpaceCreateDeviceRGB()
let bitmapInfo = (CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.premultipliedFirst.rawValue)
let context = CGContext(data: baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: bytesPerRow, space: colorSpace, bitmapInfo: bitmapInfo)
let imageRef = context!.makeImage()
CVPixelBufferUnlockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0))
return UIImage(cgImage: imageRef!)
}
// volume switch
var initialVolume = 0.0
var volumeView: MPVolumeView?
func listenVolumeButton() {
volumeView = MPVolumeView(frame: CGRect(x:-3000, y:0, width:0, height:0))
view.addSubview(volumeView!)
let audioSession = AVAudioSession.sharedInstance()
do {
try audioSession.setActive(true)
let vol = audioSession.outputVolume
initialVolume = Double(vol.description)!
if initialVolume > 0.9 {
initialVolume = 0.9
} else if initialVolume < 0.1 {
initialVolume = 0.1
}
} catch {
print("error: \(error)")
}
audioSession.addObserver(self, forKeyPath: "outputVolume", options:
NSKeyValueObservingOptions.new, context: nil)
}
func removeVolumeButton() {
AVAudioSession.sharedInstance().removeObserver(self, forKeyPath: "outputVolume")
volumeView?.removeFromSuperview()
}
override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) {
if keyPath == "outputVolume" {
let volume = (change?[NSKeyValueChangeKey.newKey] as! NSNumber).floatValue
let newVolume = Double(volume)
if newVolume > initialVolume + 0.05 {
if let view = volumeView?.subviews.first as? UISlider {
// volume up pressed
view.value = Float(initialVolume)
if zoom < 12.0 {
zoom *= 1.2
} else {
zoom = 1.0
}
print("zoom: \(zoom)")
}
} else if newVolume < initialVolume - 0.05 {
if let view = volumeView?.subviews.first as? UISlider {
// volume down pressed
view.value = Float(initialVolume)
/*
if zoom > 1.0 {
zoom /= 1.2
}
*/
if let image = self.imageView1.image {
saveImage(image: image)
print("save image")
}
}
}
}
}
// save
private func saveImage(image: UIImage) {
UIImageWriteToSavedPhotosAlbum(image, nil, nil, nil)
}
}
「megane - qrdetector on GitHub」 in Swift4
links
- WebMegane - your new digital sight / デジタルな視界 on iPhone Safari x VR goggles (JS、webアプリ版)
