/* * Copyright (c) 2015 Allan Pichardo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ using UnityEngine; using System.Collections; using System.Collections.Generic; [RequireComponent (typeof(AudioSource))] public class AudioProcessor : MonoBehaviour { public AudioSource audioSource; private long lastT, nowT, diff, entries, sum; public int bufferSize = 1024; // fft size private int samplingRate = 44100; // fft sampling frequency /* log-frequency averaging controls */ private int nBand = 12; // number of bands public float gThresh = 0.1f; // sensitivity int blipDelayLen = 16; int[] blipDelay; private int sinceLast = 0; // counter to suppress double-beats private float framePeriod; /* storage space */ private int colmax = 120; float[] spectrum; float[] averages; float[] acVals; float[] onsets; float[] scorefun; float[] dobeat; int now = 0; // time index for circular buffer within above float[] spec; // the spectrum of the previous step /* Autocorrelation structure */ int maxlag = 100; // (in frames) largest lag to track float decay = 0.997f; // smoothing constant for running average Autoco auco; private float alph; // trade-off constant between tempo deviation penalty and onset strength [Header ("Events")] public OnBeatEventHandler onBeat; public OnSpectrumEventHandler onSpectrum; ////////////////////////////////// private long getCurrentTimeMillis () { long milliseconds = System.DateTime.Now.Ticks / System.TimeSpan.TicksPerMillisecond; return milliseconds; } private void initArrays () { blipDelay = new int[blipDelayLen]; onsets = new float[colmax]; scorefun = new float[colmax]; dobeat = new float[colmax]; spectrum = new float[bufferSize]; averages = new float[12]; acVals = new float[maxlag]; alph = 100 * gThresh; } // Use this for initialization void Start () { initArrays (); audioSource = GetComponent (); samplingRate = audioSource.clip.frequency; framePeriod = (float)bufferSize / (float)samplingRate; //initialize record of previous spectrum spec = new float[nBand]; for (int i = 0; i < nBand; ++i) spec [i] = 100.0f; auco = new Autoco (maxlag, decay, framePeriod, getBandWidth ()); lastT = getCurrentTimeMillis (); } public void tapTempo () { nowT = getCurrentTimeMillis (); diff = nowT - lastT; lastT = nowT; sum = sum + diff; entries++; int average = (int)(sum / entries); Debug.Log ("average = " + average); } double[] toDoubleArray (float[] arr) { if (arr == null) return null; int n = arr.Length; double[] ret = new double[n]; for (int i = 0; i < n; i++) { ret [i] = (float)arr [i]; } return ret; } // Update is called once per frame void Update () { if (audioSource.isPlaying) { audioSource.GetSpectrumData (spectrum, 0, FFTWindow.BlackmanHarris); computeAverages (spectrum); onSpectrum.Invoke (averages); /* calculate the value of the onset function in this frame */ float onset = 0; for (int i = 0; i < nBand; i++) { float specVal = (float)System.Math.Max (-100.0f, 20.0f * (float)System.Math.Log10 (averages [i]) + 160); // dB value of this band specVal *= 0.025f; float dbInc = specVal - spec [i]; // dB increment since last frame spec [i] = specVal; // record this frome to use next time around onset += dbInc; // onset function is the sum of dB increments } onsets [now] = onset; /* update autocorrelator and find peak lag = current tempo */ auco.newVal (onset); // record largest value in (weighted) autocorrelation as it will be the tempo float aMax = 0.0f; int tempopd = 0; //float[] acVals = new float[maxlag]; for (int i = 0; i < maxlag; ++i) { float acVal = (float)System.Math.Sqrt (auco.autoco (i)); if (acVal > aMax) { aMax = acVal; tempopd = i; } // store in array backwards, so it displays right-to-left, in line with traces acVals [maxlag - 1 - i] = acVal; } /* calculate DP-ish function to update the best-score function */ float smax = -999999; int smaxix = 0; // weight can be varied dynamically with the mouse alph = 100 * gThresh; // consider all possible preceding beat times from 0.5 to 2.0 x current tempo period for (int i = tempopd / 2; i < System.Math.Min (colmax, 2 * tempopd); ++i) { // objective function - this beat's cost + score to last beat + transition penalty float score = onset + scorefun [(now - i + colmax) % colmax] - alph * (float)System.Math.Pow (System.Math.Log ((float)i / (float)tempopd), 2); // keep track of the best-scoring predecesor if (score > smax) { smax = score; smaxix = i; } } scorefun [now] = smax; // keep the smallest value in the score fn window as zero, by subtracing the min val float smin = scorefun [0]; for (int i = 0; i < colmax; ++i) if (scorefun [i] < smin) smin = scorefun [i]; for (int i = 0; i < colmax; ++i) scorefun [i] -= smin; /* find the largest value in the score fn window, to decide if we emit a blip */ smax = scorefun [0]; smaxix = 0; for (int i = 0; i < colmax; ++i) { if (scorefun [i] > smax) { smax = scorefun [i]; smaxix = i; } } // dobeat array records where we actally place beats dobeat [now] = 0; // default is no beat this frame ++sinceLast; // if current value is largest in the array, probably means we're on a beat if (smaxix == now) { //tapTempo(); // make sure the most recent beat wasn't too recently if (sinceLast > tempopd / 4) { onBeat.Invoke (); blipDelay [0] = 1; // record that we did actually mark a beat this frame dobeat [now] = 1; // reset counter of frames since last beat sinceLast = 0; } } /* update column index (for ring buffer) */ if (++now == colmax) now = 0; //Debug.Log(System.Math.Round(60 / (tempopd * framePeriod)) + " bpm"); //Debug.Log(System.Math.Round(auco.avgBpm()) + " bpm"); } } public void changeCameraColor () { //Debug.Log("camera"); float r = Random.Range (0f, 1f); float g = Random.Range (0f, 1f); float b = Random.Range (0f, 1f); //Debug.Log(r + "," + g + "," + b); Color color = new Color (r, g, b); GetComponent ().clearFlags = CameraClearFlags.Color; Camera.main.backgroundColor = color; //camera.backgroundColor = color; } public float getBandWidth () { return (2f / (float)bufferSize) * (samplingRate / 2f); } public int freqToIndex (int freq) { // special case: freq is lower than the bandwidth of spectrum[0] if (freq < getBandWidth () / 2) return 0; // special case: freq is within the bandwidth of spectrum[512] if (freq > samplingRate / 2 - getBandWidth () / 2) return (bufferSize / 2); // all other cases float fraction = (float)freq / (float)samplingRate; int i = (int)System.Math.Round (bufferSize * fraction); //Debug.Log("frequency: " + freq + ", index: " + i); return i; } public void computeAverages (float[] data) { for (int i = 0; i < 12; i++) { float avg = 0; int lowFreq; if (i == 0) lowFreq = 0; else lowFreq = (int)((samplingRate / 2) / (float)System.Math.Pow (2, 12 - i)); int hiFreq = (int)((samplingRate / 2) / (float)System.Math.Pow (2, 11 - i)); int lowBound = freqToIndex (lowFreq); int hiBound = freqToIndex (hiFreq); for (int j = lowBound; j <= hiBound; j++) { //Debug.Log("lowbound: " + lowBound + ", highbound: " + hiBound); avg += data [j]; } // line has been changed since discussion in the comments // avg /= (hiBound - lowBound); avg /= (hiBound - lowBound + 1); averages [i] = avg; } } float map (float s, float a1, float a2, float b1, float b2) { return b1 + (s - a1) * (b2 - b1) / (a2 - a1); } public float constrain (float value, float inclusiveMinimum, float inlusiveMaximum) { if (value >= inclusiveMinimum) { if (value <= inlusiveMaximum) { return value; } return inlusiveMaximum; } return inclusiveMinimum; } [System.Serializable] public class OnBeatEventHandler : UnityEngine.Events.UnityEvent { } [System.Serializable] public class OnSpectrumEventHandler : UnityEngine.Events.UnityEvent { } // class to compute an array of online autocorrelators private class Autoco { private int del_length; private float decay; private float[] delays; private float[] outputs; private int indx; private float[] bpms; private float[] rweight; private float wmidbpm = 120f; private float woctavewidth; public Autoco (int len, float alpha, float framePeriod, float bandwidth) { woctavewidth = bandwidth; decay = alpha; del_length = len; delays = new float[del_length]; outputs = new float[del_length]; indx = 0; // calculate a log-lag gaussian weighting function, to prefer tempi around 120 bpm bpms = new float[del_length]; rweight = new float[del_length]; for (int i = 0; i < del_length; ++i) { bpms [i] = 60.0f / (framePeriod * (float)i); //Debug.Log(bpms[i]); // weighting is Gaussian on log-BPM axis, centered at wmidbpm, SD = woctavewidth octaves rweight [i] = (float)System.Math.Exp (-0.5f * System.Math.Pow (System.Math.Log (bpms [i] / wmidbpm) / System.Math.Log (2.0f) / woctavewidth, 2.0f)); } } public void newVal (float val) { delays [indx] = val; // update running autocorrelator values for (int i = 0; i < del_length; ++i) { int delix = (indx - i + del_length) % del_length; outputs [i] += (1 - decay) * (delays [indx] * delays [delix] - outputs [i]); } if (++indx == del_length) indx = 0; } // read back the current autocorrelator value at a particular lag public float autoco (int del) { float blah = rweight [del] * outputs [del]; return blah; } public float avgBpm () { float sum = 0; for (int i = 0; i < bpms.Length; ++i) { sum += bpms [i]; } return sum / del_length; } } }