From 19e6b0450a7096c2d277c9003d56d49381234713 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Tue, 1 Mar 2016 11:54:16 +1100 Subject: [PATCH] Basic voice control implementation --- src/inject/GPMInject/interface/index.js | 1 + .../voiceControls/SpeechRecognizer.js | 96 +++++++++++ .../interface/voiceControls/index.js | 157 ++++++++++++++++++ .../voiceControls/utils/playPlaylist.js | 32 ++++ .../interface/voiceControls/utils/waitFor.js | 10 ++ 5 files changed, 296 insertions(+) create mode 100644 src/inject/GPMInject/interface/voiceControls/SpeechRecognizer.js create mode 100644 src/inject/GPMInject/interface/voiceControls/index.js create mode 100644 src/inject/GPMInject/interface/voiceControls/utils/playPlaylist.js create mode 100644 src/inject/GPMInject/interface/voiceControls/utils/waitFor.js diff --git a/src/inject/GPMInject/interface/index.js b/src/inject/GPMInject/interface/index.js index ff42ec15..1091cd79 100644 --- a/src/inject/GPMInject/interface/index.js +++ b/src/inject/GPMInject/interface/index.js @@ -2,3 +2,4 @@ import './customUI'; import './customTheme'; import './errorHandler'; import './mini'; +import './voiceControls'; diff --git a/src/inject/GPMInject/interface/voiceControls/SpeechRecognizer.js b/src/inject/GPMInject/interface/voiceControls/SpeechRecognizer.js new file mode 100644 index 00000000..1d51c75f --- /dev/null +++ b/src/inject/GPMInject/interface/voiceControls/SpeechRecognizer.js @@ -0,0 +1,96 @@ +import _ from 'lodash'; + +const COMMAND_CONNECTORS = ['and then', 'and', 'then']; + +export default class SpeechRecognizer { + constructor(hotwords = [], prefixes = []) { + this.hotwords = hotwords; + this.prefixes = prefixes; + this.speech = new window.webkitSpeechRecognition(); // eslint-disable-line + + this.speech.onresult = this._onSpeech.bind(this); + + this.speech.onerror = () => this.speech.stop(); + this.speech.onnomatch = () => this.speech.stop(); + this.speech.onend = () => { + if (Settings.get('speechControl', true)) { + this.speech.start(); + } + }; + + this.speech.start(); + + this._handlers = {}; + } + + _onSpeech(event) { + const results = event.results; + if (results['0'] && results['0']['0']) { + const said = results['0']['0'].transcript.trim(); + _.forEach(this.hotwords, (hotword) => { + if (said.substr(0, hotword.length) === hotword) { + this.handleCommand(said.substr(hotword.length).trim()); + return false; + } + }); + } + this.speech.stop(); + } + + handleCommand(command) { + if (!command) return; + let matchedFn; + let matchedKey = ''; + let fnArg; + + _.forIn(this._handlers, (fn, key) => { + if (command.substr(0, key.length).toLowerCase() === key) { + if (key.length > matchedKey.length) { + matchedKey = key; + matchedFn = fn; + fnArg = command.substr(key.length).trim(); + } + } + }); + + if (matchedFn) { + matchedFn(fnArg) + .catch((error) => { + console.info(error); + }) + .then((unusedCommand) => { + if (unusedCommand === undefined) { + unusedCommand = fnArg; // eslint-disable-line + } + let handled = false; + _.forEach(COMMAND_CONNECTORS, (connector) => { + if (unusedCommand.substr(0, connector.length) === connector) { + handled = true; + this.handleCommand(unusedCommand.substr(connector.length).trim()); + return false; + } + }); + if (!handled) { + this.handleCommand(unusedCommand.trim()); + } + }); + } else { + console.warn(`Command: "${command}" did not match any registered handlers`); + } + } + + registerHandler(action, fn) { + let actions = action; + if (!_.isArray(actions)) actions = [action]; + + _.forEach(actions, (actionString) => { + if (this._handlers[actionString.toLowerCase()]) { + console.error(`The action "${action.toLowerCase()}" already has a handler registered`); // eslint-disable-line + } else { + _.forEach(this.prefixes.concat(['']), (prefix) => { + this._handlers[(prefix.toLowerCase() + ' ' + actionString.toLowerCase()).trim()] = fn; + }); + } + }); + } +} diff --git a/src/inject/GPMInject/interface/voiceControls/index.js b/src/inject/GPMInject/interface/voiceControls/index.js new file mode 100644 index 00000000..ce515bc8 --- /dev/null +++ b/src/inject/GPMInject/interface/voiceControls/index.js @@ -0,0 +1,157 @@ +import SpeechRecognizer from './SpeechRecognizer'; + +import playPlaylist from './utils/playPlaylist'; + +window.wait(() => { + const speech = new SpeechRecognizer( + ['ok player', 'okayplayer', 'hey music', 'yo music'], // Hot Words + ['Let\'s', 'can you', 'can you please', 'please'] // Command prefix + ); + + // Play Playlist Handlers + speech.registerHandler(['play playlist', 'play the playlist'], playPlaylist); + + // Play / Pause Handlers + let playing = false; + GPM.on('change:playback', (mode) => playing = (mode === 2)); + speech.registerHandler(['pause', 'pours', 'paws', 'Paul\'s'], () => + new Promise((resolve) => { + if (playing) GPM.playback.playPause(); + resolve(); + }) + ); + speech.registerHandler('play', () => + new Promise((resolve) => { + if (!playing) GPM.playback.playPause(); + resolve(); + }) + ); + + // Track Navigation Handlers + speech.registerHandler(['next', 'forward', 'fast forward'], () => + new Promise((resolve) => { + GPM.playback.forward(); + resolve(); + }) + ); + + speech.registerHandler(['back', 'previous', 'rewind', 'start again'], () => + new Promise((resolve) => { + GPM.playback.rewind(); + resolve(); + }) + ); + + speech.registerHandler(['this song sucks', 'the song sucks'], () => + new Promise((resolve) => { + GPM.rating.setRating(1); + resolve(); + }) + ); + + // Shuffle Handlers + speech.registerHandler(['mixitup', 'mix it up', + 'shuffle', 'shake', 'random'], () => + new Promise((resolve) => { + if (GPM.playback.getShuffle() === window.GMusic.Playback.ALL_SHUFFLE) { + GPM.playback.toggleShuffle(); + } + GPM.playback.toggleShuffle(); + GPM.playback.forward(); + resolve(); + }) + ); + + speech.registerHandler(['turn shuffle on', 'shuffle on'], () => + new Promise((resolve) => { + if (GPM.playback.getShuffle() === window.GMusic.Playback.NO_SHUFFLE) { + GPM.playback.toggleShuffle(); + } + resolve(); + }) + ); + + speech.registerHandler(['turn shuffle off', 'shuffle off'], () => + new Promise((resolve) => { + if (GPM.playback.getShuffle() === window.GMusic.Playback.ALL_SHUFFLE) { + GPM.playback.toggleShuffle(); + } + resolve(); + }) + ); + + // Album Navigation Handlers + speech.registerHandler(['goto artist', 'go to artist', 'load artist', + 'navigate to artist'], (artistName) => + new Promise((resolve) => { + if (!artistName) { resolve(); return; } + window.location = `/music/listen#/artist//${artistName}`; + resolve(''); + }) + ); + + // Desktop Settings Trigger + speech.registerHandler(['settings', 'open settings', 'show settings', + 'load settings'], () => + new Promise((resolve) => { + Emitter.fire('window:settings'); + resolve(); + }) + ); + + // Volume Controls + speech.registerHandler(['turn it up', 'bring it up', 'turn the volume up', + 'make it louder', 'i can\'t here it'], () => + new Promise((resolve) => { + GPM.volume.increaseVolume(); + resolve(); + }) + ); + + speech.registerHandler(['turn it down', 'take it down', 'turn the volume down', + 'make it quieter', 'make it more quiet'], () => + new Promise((resolve) => { + GPM.volume.decreaseVolume(); + resolve(); + }) + ); + + speech.registerHandler(['set volume to', 'set the volume to', + 'make the volume', 'set the volume 2', + 'set volume 2', 'set volume'], (num) => + new Promise((resolve) => { + const targetVol = parseInt(num, 10); + if (targetVol) { + GPM.volume.setVolume(Math.min(Math.max(0, targetVol), 100)); + } + resolve(num.replace(new RegExp(targetVol.toString() + '%', 'g'), '').trim()); + }) + ); + + let origVolume; + speech.registerHandler(['make it boom', 'make it burn', 'sing it out', 'pump it', + 'get this party started'], () => + new Promise((resolve) => { + origVolume = origVolume || GPM.volume.getVolume(); + GPM.volume.setVolume(100); + resolve(); + }) + ); + + speech.registerHandler(['shut up', 'mute', 'silence', 'turn this right down', + 'party is over', 'party\'s over'], () => + new Promise((resolve) => { + origVolume = origVolume || GPM.volume.getVolume(); + GPM.volume.setVolume(0); + resolve(); + }) + ); + + speech.registerHandler(['reset the volume', 'normalize', 'normalise'], () => + new Promise((resolve) => { + GPM.volume.setVolume(origVolume); + origVolume = null; + resolve(); + }) + ); +}); diff --git a/src/inject/GPMInject/interface/voiceControls/utils/playPlaylist.js b/src/inject/GPMInject/interface/voiceControls/utils/playPlaylist.js new file mode 100644 index 00000000..012a5dcf --- /dev/null +++ b/src/inject/GPMInject/interface/voiceControls/utils/playPlaylist.js @@ -0,0 +1,32 @@ +import _ from 'lodash'; + +import waitFor from './waitFor'; + +export default function (playlistName) { + return new Promise((resolve, reject) => { + if (!playlistName) { resolve(); return; } + const playlistLinks = document.querySelectorAll('#playlists > a'); + let foundLink; + + _.forEach(playlistLinks, (link) => { + const label = link.querySelector('div').innerText; + if (label.toLowerCase().trim() === playlistName.toLowerCase()) { + foundLink = link; + } + }); + + if (foundLink) { + foundLink.click(); + waitFor(() => { + const title = document.querySelector('.playlist-view .material-container-details .info .title'); // eslint-disable-line + return title && title.innerText.toLowerCase().trim() === playlistName.toLowerCase(); + }) + .then(() => { + document.querySelector('.playlist-view .material-container-details [data-id=play]').click(); // eslint-disable-line + resolve(''); + }); + } else { + reject(`There is no playlist with the name ${playlistName}`); + } + }); +} diff --git a/src/inject/GPMInject/interface/voiceControls/utils/waitFor.js b/src/inject/GPMInject/interface/voiceControls/utils/waitFor.js new file mode 100644 index 00000000..e522d055 --- /dev/null +++ b/src/inject/GPMInject/interface/voiceControls/utils/waitFor.js @@ -0,0 +1,10 @@ +export default function (fn) { + return new Promise((resolve) => { + const wait = setInterval(() => { + if (fn()) { + clearInterval(wait); + resolve(); + } + }, 10); + }); +}