Incorrect position of anchor point in CGPointApplyAffineTransform (iOS/Swift)

It seems that the anchor point of my buttonView is changing based on the position of view. I want the buttonView to shrink to the middle of itself rather than the middle of view. I was wondering what I'm doing wrong? Or if I'm completely misunderstanding anchor points? I tried reading some articles but they seemed to suggest that the anchor point of an object referred to its own coordinates rather than its super view's.

import UIKit

class ViewController: UIViewController {
    var buttonView:UIView!
    var toggleButton:UIButton!
    var isVisible = false

    override func viewDidLoad() {
        super.viewDidLoad()
        // Do any additional setup after loading the view, typically from a nib.

        let buttonTitles = ["1", "2", "3"]
        let size:CGFloat = 100.0

        let viewSize = CGFloat(buttonTitles.count) * size
        buttonView = UIView(frame: CGRectMake(40, 40, viewSize, viewSize))

        setAnchorPoint(CGPointMake(0.5, 0), forView: buttonView)
        buttonView.layer.anchorPoint = CGPointMake(0.5, 0.0)
        view.addSubview(buttonView)

        for (index, title) in buttonTitles.enumerate() {
            let button = UIButton(type: .Custom) as UIButton
            button.frame = CGRectMake(0, size * CGFloat(index), size, size)
            button.setTitle(title, forState: .Normal)
            button.backgroundColor = UIColor.redColor()
            buttonView.addSubview(button)
        }

        toggleButton = UIButton(type: .Custom) as UIButton
        toggleButton.frame = CGRectMake(200, 100, 80, 50)
        toggleButton.setTitle("Toggle", forState: .Normal)
        toggleButton.backgroundColor = UIColor.purpleColor()
        toggleButton.addTarget(self, action: #selector(ViewController.animate), forControlEvents: .TouchUpInside)
        view.addSubview(toggleButton)
    }

    func setAnchorPoint(anchorPoint: CGPoint, forView view: UIView) {
        var newPoint = CGPointMake(view.bounds.size.width * anchorPoint.x, view.bounds.size.height * anchorPoint.y)
        var oldPoint = CGPointMake(view.bounds.size.width * view.layer.anchorPoint.x, view.bounds.size.height * view.layer.anchorPoint.y)

        newPoint = CGPointApplyAffineTransform(newPoint, view.transform)
        oldPoint = CGPointApplyAffineTransform(oldPoint, view.transform)

        var position = view.layer.position
        position.x -= oldPoint.x
        position.x += newPoint.x

        position.y -= oldPoint.y
        position.y += newPoint.y

        view.layer.position = position
        view.layer.anchorPoint = anchorPoint
    }

    func animate() {

        if (isVisible) { // collapse it
            isVisible = false

            UIView.animateWithDuration(0.5, animations: { 
                self.buttonView.transform = CGAffineTransformMakeScale(0.2, 0.2)
            })
        }

        else { // expand it
            isVisible = true

            UIView.animateWithDuration(0.5, animations: {
                self.buttonView.transform = CGAffineTransformMakeScale(1.0, 1.0)
            })
        }
    }   
}

Before clicking Toggle:

After clicking Toggle:

Nevermind, turns out I just set my buttonView's width too big.