libs/rubberband/src/vamp/RubberBandVampPlugin.cpp

   1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
   2
   3 /*
   4     Rubber Band
   5     An audio time-stretching and pitch-shifting library.
   6     Copyright 2007 Chris Cannam.
   7
   8     This program is free software; you can redistribute it and/or
   9     modify it under the terms of the GNU General Public License as
  10     published by the Free Software Foundation; either version 2 of the
  11     License, or (at your option) any later version.  See the file
  12     COPYING included with this distribution for more information.
  13 */
  14
  15 #include "RubberBandVampPlugin.h"
  16
  17 #include "StretchCalculator.h"
  18
  19 #include <cmath>
  20
  21 using std::string;
  22 using std::vector;
  23 using std::cerr;
  24 using std::endl;
  25
  26 class RubberBandVampPlugin::Impl
  27 {
  28 public:
  29     size_t m_stepSize;
  30     size_t m_blockSize;
  31     size_t m_sampleRate;
  32
  33     float m_timeRatio;
  34     float m_pitchRatio;
  35
  36     bool m_realtime;
  37     bool m_elasticTiming;
  38     int m_transientMode;
  39     bool m_phaseIndependent;
  40     int m_windowLength;
  41
  42     RubberBand::RubberBandStretcher *m_stretcher;
  43
  44     int m_incrementsOutput;
  45     int m_aggregateIncrementsOutput;
  46     int m_divergenceOutput;
  47     int m_phaseResetDfOutput;
  48     int m_smoothedPhaseResetDfOutput;
  49     int m_phaseResetPointsOutput;
  50     int m_timeSyncPointsOutput;
  51
  52     size_t m_counter;
  53     size_t m_accumulatedIncrement;
  54
  55     float **m_outputDump;
  56
  57     FeatureSet processOffline(const float *const *inputBuffers,
  58                               Vamp::RealTime timestamp);
  59
  60     FeatureSet getRemainingFeaturesOffline();
  61
  62     FeatureSet processRealTime(const float *const *inputBuffers,
  63                                Vamp::RealTime timestamp);
  64
  65     FeatureSet getRemainingFeaturesRealTime();
  66
  67     FeatureSet createFeatures(size_t inputIncrement,
  68                               std::vector<int> &outputIncrements,
  69                               std::vector<float> &phaseResetDf,
  70                               std::vector<int> &exactPoints,
  71                               std::vector<float> &smoothedDf,
  72                               size_t baseCount,
  73                               bool includeFinal);
  74 };
  75
  76
  77 RubberBandVampPlugin::RubberBandVampPlugin(float inputSampleRate) :
  78     Plugin(inputSampleRate)
  79 {
  80     m_d = new Impl();
  81     m_d->m_stepSize = 0;
  82     m_d->m_timeRatio = 1.f;
  83     m_d->m_pitchRatio = 1.f;
  84     m_d->m_realtime = false;
  85     m_d->m_elasticTiming = true;
  86     m_d->m_transientMode = 0;
  87     m_d->m_phaseIndependent = false;
  88     m_d->m_windowLength = 0;
  89     m_d->m_stretcher = 0;
  90     m_d->m_sampleRate = lrintf(m_inputSampleRate);
  91 }
  92
  93 RubberBandVampPlugin::~RubberBandVampPlugin()
  94 {
  95     if (m_d->m_outputDump) {
  96         for (size_t i = 0; i < m_d->m_stretcher->getChannelCount(); ++i) {
  97             delete[] m_d->m_outputDump[i];
  98         }
  99         delete[] m_d->m_outputDump;
 100     }
 101     delete m_d->m_stretcher;
 102     delete m_d;
 103 }
 104
 105 string
 106 RubberBandVampPlugin::getIdentifier() const
 107 {
 108     return "rubberband";
 109 }
 110
 111 string
 112 RubberBandVampPlugin::getName() const
 113 {
 114     return "Rubber Band Timestretch Analysis";
 115 }
 116
 117 string
 118 RubberBandVampPlugin::getDescription() const
 119 {
 120     return "Carry out analysis phases of time stretcher process";
 121 }
 122
 123 string
 124 RubberBandVampPlugin::getMaker() const
 125 {
 126     return "Rubber Band"; ///!!!
 127 }
 128
 129 int
 130 RubberBandVampPlugin::getPluginVersion() const
 131 {
 132     return 1;
 133 }
 134
 135 string
 136 RubberBandVampPlugin::getCopyright() const
 137 {
 138     return "";//!!!
 139 }
 140
 141 RubberBandVampPlugin::OutputList
 142 RubberBandVampPlugin::getOutputDescriptors() const
 143 {
 144     OutputList list;
 145
 146     size_t rate = 0;
 147     if (m_d->m_stretcher) {
 148         rate = lrintf(m_inputSampleRate / m_d->m_stretcher->getInputIncrement());
 149     }
 150
 151     OutputDescriptor d;
 152     d.identifier = "increments";
 153     d.name = "Output Increments";
 154     d.description = "Output time increment for each input step";
 155     d.unit = "samples";
 156     d.hasFixedBinCount = true;
 157     d.binCount = 1;
 158     d.hasKnownExtents = false;
 159     d.isQuantized = true;
 160     d.quantizeStep = 1.0;
 161     d.sampleType = OutputDescriptor::VariableSampleRate;
 162     d.sampleRate = rate;
 163     m_d->m_incrementsOutput = list.size();
 164     list.push_back(d);
 165
 166     d.identifier = "aggregate_increments";
 167     d.name = "Accumulated Output Increments";
 168     d.description = "Accumulated output time increments";
 169     d.sampleRate = 0;
 170     m_d->m_aggregateIncrementsOutput = list.size();
 171     list.push_back(d);
 172
 173     d.identifier = "divergence";
 174     d.name = "Divergence from Linear";
 175     d.description = "Difference between actual output time and the output time for a theoretical linear stretch";
 176     d.isQuantized = false;
 177     d.sampleRate = 0;
 178     m_d->m_divergenceOutput = list.size();
 179     list.push_back(d);
 180
 181     d.identifier = "phaseresetdf";
 182     d.name = "Phase Reset Detection Function";
 183     d.description = "Curve whose peaks are used to identify transients for phase reset points";
 184     d.unit = "";
 185     d.sampleRate = rate;
 186     m_d->m_phaseResetDfOutput = list.size();
 187     list.push_back(d);
 188
 189     d.identifier = "smoothedphaseresetdf";
 190     d.name = "Smoothed Phase Reset Detection Function";
 191     d.description = "Phase reset curve smoothed for peak picking";
 192     d.unit = "";
 193     m_d->m_smoothedPhaseResetDfOutput = list.size();
 194     list.push_back(d);
 195
 196     d.identifier = "phaseresetpoints";
 197     d.name = "Phase Reset Points";
 198     d.description = "Points estimated as transients at which phase reset occurs";
 199     d.unit = "";
 200     d.hasFixedBinCount = true;
 201     d.binCount = 0;
 202     d.hasKnownExtents = false;
 203     d.isQuantized = false;
 204     d.sampleRate = 0;
 205     m_d->m_phaseResetPointsOutput = list.size();
 206     list.push_back(d);
 207
 208     d.identifier = "timesyncpoints";
 209     d.name = "Time Sync Points";
 210     d.description = "Salient points which stretcher aims to place with strictly correct timing";
 211     d.unit = "";
 212     d.hasFixedBinCount = true;
 213     d.binCount = 0;
 214     d.hasKnownExtents = false;
 215     d.isQuantized = false;
 216     d.sampleRate = 0;
 217     m_d->m_timeSyncPointsOutput = list.size();
 218     list.push_back(d);
 219
 220     return list;
 221 }
 222
 223 RubberBandVampPlugin::ParameterList
 224 RubberBandVampPlugin::getParameterDescriptors() const
 225 {
 226     ParameterList list;
 227
 228     ParameterDescriptor d;
 229     d.identifier = "timeratio";
 230     d.name = "Time Ratio";
 231     d.description = "Ratio to modify overall duration by";
 232     d.unit = "%";
 233     d.minValue = 1;
 234     d.maxValue = 500;
 235     d.defaultValue = 100;
 236     d.isQuantized = false;
 237     list.push_back(d);
 238
 239     d.identifier = "pitchratio";
 240     d.name = "Pitch Scale Ratio";
 241     d.description = "Frequency ratio to modify pitch by";
 242     d.unit = "%";
 243     d.minValue = 1;
 244     d.maxValue = 500;
 245     d.defaultValue = 100;
 246     d.isQuantized = false;
 247     list.push_back(d);
 248
 249     d.identifier = "mode";
 250     d.name = "Processing Mode";
 251     d.description = ""; //!!!
 252     d.unit = "";
 253     d.minValue = 0;
 254     d.maxValue = 1;
 255     d.defaultValue = 0;
 256     d.isQuantized = true;
 257     d.quantizeStep = 1;
 258     d.valueNames.clear();
 259     d.valueNames.push_back("Offline");
 260     d.valueNames.push_back("Real Time");
 261     list.push_back(d);
 262
 263     d.identifier = "stretchtype";
 264     d.name = "Stretch Flexibility";
 265     d.description = ""; //!!!
 266     d.unit = "";
 267     d.minValue = 0;
 268     d.maxValue = 1;
 269     d.defaultValue = 0;
 270     d.isQuantized = true;
 271     d.quantizeStep = 1;
 272     d.valueNames.clear();
 273     d.valueNames.push_back("Elastic");
 274     d.valueNames.push_back("Precise");
 275     list.push_back(d);
 276
 277     d.identifier = "transientmode";
 278     d.name = "Transient Handling";
 279     d.description = ""; //!!!
 280     d.unit = "";
 281     d.minValue = 0;
 282     d.maxValue = 2;
 283     d.defaultValue = 0;
 284     d.isQuantized = true;
 285     d.quantizeStep = 1;
 286     d.valueNames.clear();
 287     d.valueNames.push_back("Mixed");
 288     d.valueNames.push_back("Smooth");
 289     d.valueNames.push_back("Crisp");
 290     list.push_back(d);
 291
 292     d.identifier = "phasemode";
 293     d.name = "Phase Handling";
 294     d.description = ""; //!!!
 295     d.unit = "";
 296     d.minValue = 0;
 297     d.maxValue = 1;
 298     d.defaultValue = 0;
 299     d.isQuantized = true;
 300     d.quantizeStep = 1;
 301     d.valueNames.clear();
 302     d.valueNames.push_back("Peak Locked");
 303     d.valueNames.push_back("Independent");
 304     list.push_back(d);
 305
 306     d.identifier = "windowmode";
 307     d.name = "Window Length";
 308     d.description = ""; //!!!
 309     d.unit = "";
 310     d.minValue = 0;
 311     d.maxValue = 2;
 312     d.defaultValue = 0;
 313     d.isQuantized = true;
 314     d.quantizeStep = 1;
 315     d.valueNames.clear();
 316     d.valueNames.push_back("Standard");
 317     d.valueNames.push_back("Short");
 318     d.valueNames.push_back("Long");
 319     list.push_back(d);
 320
 321     return list;
 322 }
 323
 324 float
 325 RubberBandVampPlugin::getParameter(std::string id) const
 326 {
 327     if (id == "timeratio") return m_d->m_timeRatio * 100.f;
 328     if (id == "pitchratio") return m_d->m_pitchRatio * 100.f;
 329     if (id == "mode") return m_d->m_realtime ? 1 : 0;
 330     if (id == "stretchtype") return m_d->m_elasticTiming ? 0 : 1;
 331     if (id == "transientmode") return m_d->m_transientMode;
 332     if (id == "phasemode") return m_d->m_phaseIndependent ? 1 : 0;
 333     if (id == "windowmode") return m_d->m_windowLength;
 334     return 0.f;
 335 }
 336
 337 void
 338 RubberBandVampPlugin::setParameter(std::string id, float value)
 339 {
 340     if (id == "timeratio") {
 341         m_d->m_timeRatio = value / 100;
 342     } else if (id == "pitchratio") {
 343         m_d->m_pitchRatio = value / 100;
 344     } else {
 345         bool set = (value > 0.5);
 346         if (id == "mode") m_d->m_realtime = set;
 347         else if (id == "stretchtype") m_d->m_elasticTiming = !set;
 348         else if (id == "transientmode") m_d->m_transientMode = int(value + 0.5);
 349         else if (id == "phasemode") m_d->m_phaseIndependent = set;
 350         else if (id == "windowmode") m_d->m_windowLength = int(value + 0.5);
 351     }
 352 }
 353
 354 bool
 355 RubberBandVampPlugin::initialise(size_t channels, size_t stepSize, size_t blockSize)
 356 {
 357     if (channels < getMinChannelCount() ||
 358         channels > getMaxChannelCount()) return false;
 359
 360     m_d->m_stepSize = std::min(stepSize, blockSize);
 361     m_d->m_blockSize = stepSize;
 362
 363     RubberBand::RubberBandStretcher::Options options = 0;
 364
 365     if (m_d->m_realtime)
 366          options |= RubberBand::RubberBandStretcher::OptionProcessRealTime;
 367     else options |= RubberBand::RubberBandStretcher::OptionProcessOffline;
 368
 369     if (m_d->m_elasticTiming)
 370          options |= RubberBand::RubberBandStretcher::OptionStretchElastic;
 371     else options |= RubberBand::RubberBandStretcher::OptionStretchPrecise;
 372
 373     if (m_d->m_transientMode == 0)
 374          options |= RubberBand::RubberBandStretcher::OptionTransientsMixed;
 375     else if (m_d->m_transientMode == 1)
 376          options |= RubberBand::RubberBandStretcher::OptionTransientsSmooth;
 377     else options |= RubberBand::RubberBandStretcher::OptionTransientsCrisp;
 378
 379     if (m_d->m_phaseIndependent)
 380          options |= RubberBand::RubberBandStretcher::OptionPhaseIndependent;
 381     else options |= RubberBand::RubberBandStretcher::OptionPhasePeakLocked;
 382
 383     if (m_d->m_windowLength == 0)
 384          options |= RubberBand::RubberBandStretcher::OptionWindowStandard;
 385     else if (m_d->m_windowLength == 1)
 386          options |= RubberBand::RubberBandStretcher::OptionWindowShort;
 387     else options |= RubberBand::RubberBandStretcher::OptionWindowLong;
 388
 389     delete m_d->m_stretcher;
 390     m_d->m_stretcher = new RubberBand::RubberBandStretcher
 391         (m_d->m_sampleRate, channels, options);
 392     m_d->m_stretcher->setDebugLevel(1);
 393     m_d->m_stretcher->setTimeRatio(m_d->m_timeRatio);
 394     m_d->m_stretcher->setPitchScale(m_d->m_pitchRatio);
 395
 396     m_d->m_counter = 0;
 397     m_d->m_accumulatedIncrement = 0;
 398
 399     m_d->m_outputDump = 0;
 400
 401     return true;
 402 }
 403
 404 void
 405 RubberBandVampPlugin::reset()
 406 {
 407 //    delete m_stretcher;  //!!! or just if (m_stretcher) m_stretcher->reset();
 408 //    m_stretcher = new RubberBand::RubberBandStretcher(lrintf(m_inputSampleRate), channels);
 409     if (m_d->m_stretcher) m_d->m_stretcher->reset();
 410 }
 411
 412 RubberBandVampPlugin::FeatureSet
 413 RubberBandVampPlugin::process(const float *const *inputBuffers,
 414                               Vamp::RealTime timestamp)
 415 {
 416     if (m_d->m_realtime) {
 417         return m_d->processRealTime(inputBuffers, timestamp);
 418     } else {
 419         return m_d->processOffline(inputBuffers, timestamp);
 420     }
 421 }
 422
 423 RubberBandVampPlugin::FeatureSet
 424 RubberBandVampPlugin::getRemainingFeatures()
 425 {
 426     if (m_d->m_realtime) {
 427         return m_d->getRemainingFeaturesRealTime();
 428     } else {
 429         return m_d->getRemainingFeaturesOffline();
 430     }
 431 }
 432
 433 RubberBandVampPlugin::FeatureSet
 434 RubberBandVampPlugin::Impl::processOffline(const float *const *inputBuffers,
 435                                            Vamp::RealTime timestamp)
 436 {
 437     if (!m_stretcher) {
 438         cerr << "ERROR: RubberBandVampPlugin::processOffline: "
 439              << "RubberBandVampPlugin has not been initialised"
 440              << endl;
 441         return FeatureSet();
 442     }
 443
 444     m_stretcher->study(inputBuffers, m_blockSize, false);
 445     return FeatureSet();
 446 }
 447
 448 RubberBandVampPlugin::FeatureSet
 449 RubberBandVampPlugin::Impl::getRemainingFeaturesOffline()
 450 {
 451     m_stretcher->study(0, 0, true);
 452
 453     m_stretcher->calculateStretch();
 454
 455     int rate = m_sampleRate;
 456
 457     RubberBand::StretchCalculator sc(rate,
 458                                      m_stretcher->getInputIncrement(),
 459                                      true);
 460
 461     size_t inputIncrement = m_stretcher->getInputIncrement();
 462     std::vector<int> outputIncrements = m_stretcher->getOutputIncrements();
 463     std::vector<float> phaseResetDf = m_stretcher->getPhaseResetCurve();
 464     std::vector<int> peaks = m_stretcher->getExactTimePoints();
 465     std::vector<float> smoothedDf = sc.smoothDF(phaseResetDf);
 466
 467     FeatureSet features = createFeatures
 468         (inputIncrement, outputIncrements, phaseResetDf, peaks, smoothedDf,
 469          0, true);
 470
 471     return features;
 472 }
 473
 474 RubberBandVampPlugin::FeatureSet
 475 RubberBandVampPlugin::Impl::processRealTime(const float *const *inputBuffers,
 476                                             Vamp::RealTime timestamp)
 477 {
 478     // This function is not in any way a real-time function (i.e. it
 479     // has no requirement to be RT safe); it simply operates the
 480     // stretcher in RT mode.
 481
 482     if (!m_stretcher) {
 483         cerr << "ERROR: RubberBandVampPlugin::processRealTime: "
 484              << "RubberBandVampPlugin has not been initialised"
 485              << endl;
 486         return FeatureSet();
 487     }
 488
 489     m_stretcher->process(inputBuffers, m_blockSize, false);
 490
 491     size_t inputIncrement = m_stretcher->getInputIncrement();
 492     std::vector<int> outputIncrements = m_stretcher->getOutputIncrements();
 493     std::vector<float> phaseResetDf = m_stretcher->getPhaseResetCurve();
 494     std::vector<float> smoothedDf; // not meaningful in RT mode
 495     std::vector<int> dummyPoints;
 496     FeatureSet features = createFeatures
 497         (inputIncrement, outputIncrements, phaseResetDf, dummyPoints, smoothedDf,
 498          m_counter, false);
 499     m_counter += outputIncrements.size();
 500
 501     int available = 0;
 502     while ((available = m_stretcher->available()) > 0) {
 503         if (!m_outputDump) {
 504             m_outputDump = new float *[m_stretcher->getChannelCount()];
 505             for (size_t i = 0; i < m_stretcher->getChannelCount(); ++i) {
 506                 m_outputDump[i] = new float[m_blockSize];
 507             }
 508         }
 509         m_stretcher->retrieve(m_outputDump,
 510                               std::min(int(m_blockSize), available));
 511     }
 512
 513     return features;
 514 }
 515
 516 RubberBandVampPlugin::FeatureSet
 517 RubberBandVampPlugin::Impl::getRemainingFeaturesRealTime()
 518 {
 519     return FeatureSet();
 520 }
 521
 522 RubberBandVampPlugin::FeatureSet
 523 RubberBandVampPlugin::Impl::createFeatures(size_t inputIncrement,
 524                                            std::vector<int> &outputIncrements,
 525                                            std::vector<float> &phaseResetDf,
 526                                            std::vector<int> &exactPoints,
 527                                            std::vector<float> &smoothedDf,
 528                                            size_t baseCount,
 529                                            bool includeFinal)
 530 {
 531     size_t actual = m_accumulatedIncrement;
 532
 533     double overallRatio = m_timeRatio * m_pitchRatio;
 534
 535     char label[200];
 536
 537     FeatureSet features;
 538
 539     int rate = m_sampleRate;
 540
 541     size_t epi = 0;
 542
 543     for (size_t i = 0; i < outputIncrements.size(); ++i) {
 544
 545         size_t frame = (baseCount + i) * inputIncrement;
 546
 547         int oi = outputIncrements[i];
 548         bool hard = false;
 549         bool soft = false;
 550
 551         if (oi < 0) {
 552             oi = -oi;
 553             hard = true;
 554         }
 555
 556         if (epi < exactPoints.size() && int(i) == exactPoints[epi]) {
 557             soft = true;
 558             ++epi;
 559         }
 560
 561         double linear = (frame * overallRatio);
 562
 563         Vamp::RealTime t = Vamp::RealTime::frame2RealTime(frame, rate);
 564
 565         Feature feature;
 566         feature.hasTimestamp = true;
 567         feature.timestamp = t;
 568         feature.values.push_back(oi);
 569         feature.label = Vamp::RealTime::frame2RealTime(oi, rate).toText();
 570         features[m_incrementsOutput].push_back(feature);
 571
 572         feature.values.clear();
 573         feature.values.push_back(actual);
 574         feature.label = Vamp::RealTime::frame2RealTime(actual, rate).toText();
 575         features[m_aggregateIncrementsOutput].push_back(feature);
 576
 577         feature.values.clear();
 578         feature.values.push_back(actual - linear);
 579
 580         sprintf(label, "expected %ld, actual %ld, difference %ld (%s ms)",
 581                 long(linear), long(actual), long(actual - linear),
 582                 // frame2RealTime expects an integer frame number,
 583                 // hence our multiplication factor
 584                 (Vamp::RealTime::frame2RealTime
 585                  (lrintf((actual - linear) * 1000), rate) / 1000)
 586                 .toText().c_str());
 587         feature.label = label;
 588
 589         features[m_divergenceOutput].push_back(feature);
 590         actual += oi;
 591
 592         char buf[30];
 593
 594         if (i < phaseResetDf.size()) {
 595             feature.values.clear();
 596             feature.values.push_back(phaseResetDf[i]);
 597             sprintf(buf, "%d", baseCount + i);
 598             feature.label = buf;
 599             features[m_phaseResetDfOutput].push_back(feature);
 600         }
 601
 602         if (i < smoothedDf.size()) {
 603             feature.values.clear();
 604             feature.values.push_back(smoothedDf[i]);
 605             features[m_smoothedPhaseResetDfOutput].push_back(feature);
 606         }
 607
 608         if (hard) {
 609             feature.values.clear();
 610             feature.label = "Phase Reset";
 611             features[m_phaseResetPointsOutput].push_back(feature);
 612         }
 613
 614         if (hard || soft) {
 615             feature.values.clear();
 616             feature.label = "Time Sync";
 617             features[m_timeSyncPointsOutput].push_back(feature);
 618         }
 619     }
 620
 621     if (includeFinal) {
 622         Vamp::RealTime t = Vamp::RealTime::frame2RealTime
 623             (inputIncrement * (baseCount + outputIncrements.size()), rate);
 624         Feature feature;
 625         feature.hasTimestamp = true;
 626         feature.timestamp = t;
 627         feature.label = Vamp::RealTime::frame2RealTime(actual, rate).toText();
 628         feature.values.clear();
 629         feature.values.push_back(actual);
 630         features[m_aggregateIncrementsOutput].push_back(feature);
 631
 632         float linear = ((baseCount + outputIncrements.size())
 633                         * inputIncrement * overallRatio);
 634         feature.values.clear();
 635         feature.values.push_back(actual - linear);
 636         feature.label =  // see earlier comment
 637             (Vamp::RealTime::frame2RealTime //!!! update this as earlier label
 638              (lrintf((actual - linear) * 1000), rate) / 1000)
 639             .toText();
 640         features[m_divergenceOutput].push_back(feature);
 641     }
 642
 643     m_accumulatedIncrement = actual;
 644
 645     return features;
 646 }
 647