diff --git a/packages/core/src/asset/AssetType.ts b/packages/core/src/asset/AssetType.ts index 0112491e10..a09f6aed8b 100644 --- a/packages/core/src/asset/AssetType.ts +++ b/packages/core/src/asset/AssetType.ts @@ -48,5 +48,7 @@ export enum AssetType { /** Font. */ Font = "Font", /** Source Font, include ttf、 otf and woff. */ - SourceFont = "SourceFont" + SourceFont = "SourceFont", + /** AudioClip, inclue ogg, wav and mp3 */ + Audio = "Audio" } diff --git a/packages/core/src/audio/AudioClip.ts b/packages/core/src/audio/AudioClip.ts new file mode 100644 index 0000000000..1ffb83a6ab --- /dev/null +++ b/packages/core/src/audio/AudioClip.ts @@ -0,0 +1,52 @@ +import { Engine } from "../Engine"; +import { ReferResource } from "../asset/ReferResource"; + +/** + * Audio Clip + */ +export class AudioClip extends ReferResource { + /** the name of clip */ + name: string; + + private _audioBuffer: AudioBuffer; + + /** + * the number of discrete audio channels + */ + get channels(): Readonly { + return this._audioBuffer.numberOfChannels; + } + + /** + * the sample rate, in samples per second + */ + get sampleRate(): Readonly { + return this._audioBuffer.sampleRate; + } + + /** + * the duration, in seconds + */ + get duration(): Readonly { + return this._audioBuffer.duration; + } + + /** + * get the clip's audio buffer + */ + getData(): AudioBuffer { + return this._audioBuffer; + } + + /** + * set audio buffer for the clip + */ + setData(value: AudioBuffer): void { + this._audioBuffer = value; + } + + constructor(engine: Engine, name: string = null) { + super(engine); + this.name = name; + } +} diff --git a/packages/core/src/audio/AudioListener.ts b/packages/core/src/audio/AudioListener.ts new file mode 100644 index 0000000000..d0f8f77bdc --- /dev/null +++ b/packages/core/src/audio/AudioListener.ts @@ -0,0 +1,66 @@ +import { Component } from "../Component"; +import { Entity } from "../Entity"; +import { TransformModifyFlags } from "../Transform"; +import { ignoreClone } from "../clone/CloneManager"; +import { AudioManager } from "./AudioManager"; + +/** + * Audio Listener + * only one per scene + */ +export class AudioListener extends Component { + private _context: AudioContext; + + /** + * @internal + */ + constructor(entity: Entity) { + super(entity); + const gain = AudioManager.context.createGain(); + gain.connect(AudioManager.context.destination); + AudioManager.listener = gain; + + this._context = AudioManager.context; + this._onTransformChanged = this._onTransformChanged.bind(this); + this._registerEntityTransformListener(); + + this._setListenerPose(); + } + + /** + * @internal + */ + @ignoreClone + protected _onTransformChanged(type: TransformModifyFlags) { + this._setListenerPose(); + } + + /** + * @internal + */ + protected override _onDestroy(): void { + super._onDestroy(); + AudioManager.listener = null; + this.entity.transform._updateFlagManager.removeListener(this._onTransformChanged); + } + + private _registerEntityTransformListener() { + this.entity.transform._updateFlagManager.addListener(this._onTransformChanged); + } + + private _setListenerPose() { + const { position, worldUp, worldForward } = this.entity.transform; + const { listener, currentTime } = this._context; + + listener.positionX.setValueAtTime(position.x, currentTime); + listener.positionY.setValueAtTime(position.y, currentTime); + listener.positionZ.setValueAtTime(position.z, currentTime); + + listener.upX.setValueAtTime(worldUp.x, currentTime); + listener.upY.setValueAtTime(worldUp.y, currentTime); + listener.upZ.setValueAtTime(worldUp.z, currentTime); + + listener.forwardX.setValueAtTime(worldForward.x, currentTime); + listener.forwardY.setValueAtTime(worldForward.y, currentTime); + } +} diff --git a/packages/core/src/audio/AudioManager.ts b/packages/core/src/audio/AudioManager.ts new file mode 100644 index 0000000000..a248bf4e4f --- /dev/null +++ b/packages/core/src/audio/AudioManager.ts @@ -0,0 +1,48 @@ +/** + * @internal + * Audio Manager + */ +export class AudioManager { + /** @internal */ + private static _context: AudioContext; + /** @internal */ + private static _listener: GainNode; + + private static _unlocked: boolean = false; + + /** + * Audio context + */ + static get context(): AudioContext { + if (!AudioManager._context) { + AudioManager._context = new window.AudioContext(); + } + if (AudioManager._context.state !== "running") { + window.document.addEventListener("pointerdown", AudioManager._unlock, true); + } + return AudioManager._context; + } + + /** + * Audio Listener. Can only have one listener in a Scene. + */ + static get listener(): GainNode { + return AudioManager._listener; + } + + static set listener(value: GainNode) { + AudioManager._listener = value; + } + + private static _unlock(): void { + if (AudioManager._unlocked) { + return; + } + AudioManager._context.resume().then(() => { + if (AudioManager._context.state === "running") { + window.document.removeEventListener("pointerdown", AudioManager._unlock, true); + AudioManager._unlocked = true; + } + }); + } +} diff --git a/packages/core/src/audio/AudioSource.ts b/packages/core/src/audio/AudioSource.ts new file mode 100644 index 0000000000..229eb6d963 --- /dev/null +++ b/packages/core/src/audio/AudioSource.ts @@ -0,0 +1,266 @@ +import { Component } from "../Component"; +import { Entity } from "../Entity"; +import { AudioClip } from "./AudioClip"; +import { AudioManager } from "./AudioManager"; +import { deepClone, ignoreClone } from "../clone/CloneManager"; + +/** + * Audio Source Component + */ +export class AudioSource extends Component { + @ignoreClone + /** Whether the clip playing right now */ + isPlaying: Readonly; + + @ignoreClone + private _clip: AudioClip; + @deepClone + private _gainNode: GainNode; + @ignoreClone + private _sourceNode: AudioBufferSourceNode; + + @deepClone + private _startTime: number = 0; + @deepClone + private _pausedTime: number = null; + @deepClone + private _endTime: number = null; + @deepClone + private _duration: number = null; + @ignoreClone + private _absoluteStartTime: number; + + @deepClone + private _volume: number = 1; + @deepClone + private _lastVolume: number = 1; + @deepClone + private _playbackRate: number = 1; + @deepClone + private _loop: boolean = false; + + /** + * The audio cilp to play + */ + get clip(): AudioClip { + return this._clip; + } + + set clip(value: AudioClip) { + const lastClip = this._clip; + if (lastClip !== value) { + lastClip && lastClip._addReferCount(-1); + this._clip = value; + } + } + + /** + * The volume of the audio source. 1.0 is origin volume. + */ + get volume(): number { + return this._volume; + } + + set volume(value: number) { + this._volume = value; + if (this.isPlaying) { + this._gainNode.gain.setValueAtTime(value, AudioManager.context.currentTime); + } + } + + /** + * The playback speed of the audio source, 1.0 is normal playback speed. + */ + get playbackRate(): number { + return this._playbackRate; + } + + set playbackRate(value: number) { + this._playbackRate = value; + if (this.isPlaying) { + this._sourceNode.playbackRate.value = this._playbackRate; + } + } + + /** + * Mutes / Unmutes the AudioSource. + * Mute sets the volume = 0, Un-Mute restore the original volume. + */ + get mute(): boolean { + return this.volume === 0; + } + + set mute(value: boolean) { + if (value) { + this._lastVolume = this.volume; + this.volume = 0; + } else { + this.volume = this._lastVolume; + } + } + + /** + * Whether the audio clip looping. Default false. + */ + get loop(): boolean { + return this._loop; + } + + set loop(value: boolean) { + if (value !== this._loop) { + this._loop = value; + + if (this.isPlaying) { + this._setSourceNodeLoop(this._sourceNode); + } + } + } + + /** + * The time, in seconds, at which the sound should begin to play. Default 0. + */ + get startTime(): number { + return this._startTime; + } + + set startTime(value: number) { + this._startTime = value; + } + + /** + * The time, in seconds, at which the sound should stop to play. + */ + get endTime(): number { + return this._endTime; + } + + set endTime(value: number) { + this._endTime = value; + this._duration = this._endTime - this._startTime; + } + + /** + * Playback position in seconds. + */ + get time(): number { + if (this.isPlaying) { + return this._pausedTime + ? this.engine.time.elapsedTime - this._absoluteStartTime + this._pausedTime + : this.engine.time.elapsedTime - this._absoluteStartTime + this.startTime; + } + return 0; + } + + /** + * Plays the clip. + */ + play(): void { + if (!this._clip || !this.clip.duration || this.isPlaying) return; + if (this.startTime > this._clip.duration || this.startTime < 0) return; + if (this._duration && this._duration < 0) return; + + this._pausedTime = null; + this._play(this.startTime, this._duration); + } + + /** + * Stops playing the clip. + */ + stop(): void { + if (this._sourceNode && this.isPlaying) { + this._sourceNode.stop(); + } + } + + /** + * Pauses playing the clip. + */ + pause(): void { + if (this._sourceNode && this.isPlaying) { + this._pausedTime = this.time; + + this.isPlaying = false; + + this._sourceNode.disconnect(); + this._sourceNode.onended = null; + this._sourceNode = null; + } + } + + /** + * Unpause the paused playback of this AudioSource. + */ + unPause(): void { + if (!this.isPlaying && this._pausedTime) { + const duration = this.endTime ? this.endTime - this._pausedTime : null; + this._play(this._pausedTime, duration); + } + } + + /** @internal */ + constructor(entity: Entity) { + super(entity); + this._onPlayEnd = this._onPlayEnd.bind(this); + + this._gainNode = AudioManager.context.createGain(); + this._gainNode.connect(AudioManager.listener); + } + + /** + * @internal + */ + override _onEnable(): void { + this._clip && this.unPause(); + } + + /** + * @internal + */ + override _onDisable(): void { + this._clip && this.pause(); + } + + /** + * @internal + */ + protected override _onDestroy(): void { + super._onDestroy(); + if (this._clip) { + this._clip._addReferCount(-1); + this._clip = null; + } + } + + private _play(startTime: number, duration: number | null): void { + const source = AudioManager.context.createBufferSource(); + source.buffer = this._clip.getData(); + source.onended = this._onPlayEnd; + source.playbackRate.value = this._playbackRate; + + this._setSourceNodeLoop(source); + + this._gainNode.gain.setValueAtTime(this._volume, 0); + source.connect(this._gainNode); + + duration ? source.start(0, startTime, duration) : source.start(0, startTime); + + this._absoluteStartTime = this.engine.time.elapsedTime; + this._sourceNode = source; + this.isPlaying = true; + } + + private _onPlayEnd(): void { + if (!this.isPlaying) return; + this.isPlaying = false; + } + + private _setSourceNodeLoop(sourceNode: AudioBufferSourceNode) { + sourceNode.loop = this._loop; + if (this._loop) { + sourceNode.loopStart = this.startTime; + if (this.endTime) { + sourceNode.loopEnd = this.endTime; + } + } + } +} diff --git a/packages/core/src/audio/PositionalAudioSource.ts b/packages/core/src/audio/PositionalAudioSource.ts new file mode 100644 index 0000000000..1cd43caec1 --- /dev/null +++ b/packages/core/src/audio/PositionalAudioSource.ts @@ -0,0 +1,406 @@ +import { Component } from "../Component"; +import { Entity } from "../Entity"; +import { AudioClip } from "./AudioClip"; +import { AudioManager } from "./AudioManager"; +import { deepClone, ignoreClone } from "../clone/CloneManager"; +import { TransformModifyFlags } from "../Transform"; + +/** + * Positional Audio Source Component + */ +export class PositionalAudioSource extends Component { + @ignoreClone + /** Whether the clip playing right now */ + isPlaying: Readonly; + + @ignoreClone + private _clip: AudioClip; + @deepClone + private _gainNode: GainNode; + @deepClone + private _pannerNode: PannerNode; + @ignoreClone + private _sourceNode: AudioBufferSourceNode; + + @deepClone + private _startTime: number = 0; + @deepClone + private _pausedTime: number = null; + @deepClone + private _endTime: number = null; + @deepClone + private _duration: number = null; + @ignoreClone + private _absoluteStartTime: number; + + @deepClone + private _volume: number = 1; + @deepClone + private _lastVolume: number = 1; + @deepClone + private _playbackRate: number = 1; + @deepClone + private _loop: boolean = false; + + /** + * The audio cilp to play + */ + get clip(): AudioClip { + return this._clip; + } + + set clip(value: AudioClip) { + const lastClip = this._clip; + if (lastClip !== value) { + lastClip && lastClip._addReferCount(-1); + this._clip = value; + } + } + + /** + * The volume of the audio source. 1.0 is origin volume. + */ + get volume(): number { + return this._volume; + } + + set volume(value: number) { + this._volume = value; + if (this.isPlaying) { + this._gainNode.gain.setValueAtTime(value, AudioManager.context.currentTime); + } + } + + /** + * The playback speed of the audio source, 1.0 is normal playback speed. + */ + get playbackRate(): number { + return this._playbackRate; + } + + set playbackRate(value: number) { + this._playbackRate = value; + if (this.isPlaying) { + this._sourceNode.playbackRate.value = this._playbackRate; + } + } + + /** + * Mutes / Unmutes the AudioSource. + * Mute sets the volume = 0, Un-Mute restore the original volume. + */ + get mute(): boolean { + return this.volume === 0; + } + + set mute(value: boolean) { + if (value) { + this._lastVolume = this.volume; + this.volume = 0; + } else { + this.volume = this._lastVolume; + } + } + + /** + * Whether the audio clip looping. Default false. + */ + get loop(): boolean { + return this._loop; + } + + set loop(value: boolean) { + if (value !== this._loop) { + this._loop = value; + + if (this.isPlaying) { + this._setSourceNodeLoop(this._sourceNode); + } + } + } + + /** + * The time, in seconds, at which the sound should begin to play. Default 0. + */ + get startTime(): number { + return this._startTime; + } + + set startTime(value: number) { + this._startTime = value; + } + + /** + * The time, in seconds, at which the sound should stop to play. + */ + get endTime(): number { + return this._endTime; + } + + set endTime(value: number) { + this._endTime = value; + this._duration = this._endTime - this._startTime; + } + + /** + * The spatialization algorithm to use to position the audio in 3D space. + * Default equal power. + */ + get PanningMode(): PanningModelType { + return this._pannerNode.panningModel; + } + + set PanningMode(value: PanningModelType) { + this._pannerNode.panningModel = value; + } + + /** + * The algorithm to use to reduce the volume of the audio source as it moves away from the listener. + * Default inverse. + */ + get distanceModel(): DistanceModelType { + return this._pannerNode.distanceModel; + } + + set distanceModel(value: DistanceModelType) { + this._pannerNode.distanceModel = value; + } + + /** + * The minimum distance which the volume start to reduce. + * Used by all distance models. + * Defalut 1. + */ + get minDistance(): number { + return this._pannerNode.refDistance; + } + + set minDistance(value: number) { + this._pannerNode.refDistance = value; + } + /** + * The maximum distance beyond which the volume is not reduced any further. + * Only effective when distance model set to 'linear'. + * Default 10000. + */ + get maxDistance(): number { + return this._pannerNode.maxDistance; + } + + set maxDistance(value: number) { + this._pannerNode.maxDistance = value; + } + + /** + * Direcitonal audio clip. + * The angle, in degrees, of a cone inside of which there will be no volume reduction. + * Default 360. + */ + get innerAngle(): number { + return this._pannerNode.coneInnerAngle; + } + + set innerAngle(value: number) { + this._pannerNode.coneInnerAngle = value; + } + + /** + * Directional audio clip. + * The angle, in degrees, of a cone outside of which the volume will be reduced by a constant value. + * Default 360. + */ + get outerAngle(): number { + return this._pannerNode.coneOuterAngle; + } + + set outerAngle(value: number) { + this._pannerNode.coneOuterAngle = value; + } + + /** + * Directional audio clip. + * The volume outside the cone defined by the coneOuterAngle. Default 0, meaning that no sound can be heard. + * Default 0. + */ + get outerVolume(): number { + return this._pannerNode.coneOuterGain; + } + + set outerVolume(value: number) { + this._pannerNode.coneOuterGain = value; + } + + /** + * Playback position in seconds. + */ + get time(): number { + if (this.isPlaying) { + return this._pausedTime + ? this.engine.time.elapsedTime - this._absoluteStartTime + this._pausedTime + : this.engine.time.elapsedTime - this._absoluteStartTime + this.startTime; + } + return 0; + } + + /** + * Plays the clip. + */ + play(): void { + if (!this._clip || !this.clip.duration || this.isPlaying) return; + if (this.startTime > this._clip.duration || this.startTime < 0) return; + if (this._duration && this._duration < 0) return; + + this._pausedTime = null; + this._play(this.startTime, this._duration); + } + + /** + * Stops playing the clip. + */ + stop(): void { + if (this._sourceNode && this.isPlaying) { + this._sourceNode.stop(); + } + } + + /** + * Pauses playing the clip. + */ + pause(): void { + if (this._sourceNode && this.isPlaying) { + this._pausedTime = this.time; + + this.isPlaying = false; + + this._sourceNode.disconnect(); + this._sourceNode.onended = null; + this._sourceNode = null; + } + } + + /** + * Unpause the paused playback of this AudioSource. + */ + unPause(): void { + if (!this.isPlaying && this._pausedTime) { + const duration = this.endTime ? this.endTime - this._pausedTime : null; + this._play(this._pausedTime, duration); + } + } + + /** @internal */ + constructor(entity: Entity) { + super(entity); + this._onPlayEnd = this._onPlayEnd.bind(this); + + this._gainNode = AudioManager.context.createGain(); + this._pannerNode = AudioManager.context.createPanner(); + + this._pannerNode.connect(this._gainNode); + this._gainNode.connect(AudioManager.listener); + + this._onTransformChanged = this._onTransformChanged.bind(this); + this._registerEntityTransformListener(); + + this._updatePosition(); + this._updateOrientation(); + } + + /** + * @internal + */ + override _onEnable(): void { + this._clip && this.unPause(); + } + + /** + * @internal + */ + override _onDisable(): void { + this._clip && this.pause(); + } + + /** + * @internal + */ + protected override _onDestroy(): void { + super._onDestroy(); + if (this._clip) { + this._clip._addReferCount(-1); + this._clip = null; + } + + this.entity.transform._updateFlagManager.removeListener(this._onTransformChanged); + } + + /** + * @internal + */ + @ignoreClone + protected _onTransformChanged(type: TransformModifyFlags) { + if ((type & TransformModifyFlags.WmWp) != 0) { + this._updatePosition(); + } + + if ((type & TransformModifyFlags.WmWeWq) != 0) { + this._updateOrientation(); + } + } + + private _play(startTime: number, duration: number | null): void { + const source = AudioManager.context.createBufferSource(); + source.buffer = this._clip.getData(); + source.onended = this._onPlayEnd; + source.playbackRate.value = this._playbackRate; + + this._setSourceNodeLoop(source); + + this._gainNode.gain.setValueAtTime(this._volume, 0); + source.connect(this._pannerNode); + + duration ? source.start(0, startTime, duration) : source.start(0, startTime); + + this._absoluteStartTime = this.engine.time.elapsedTime; + this._sourceNode = source; + this.isPlaying = true; + } + + private _onPlayEnd(): void { + if (!this.isPlaying) return; + this.isPlaying = false; + } + + private _setSourceNodeLoop(sourceNode: AudioBufferSourceNode) { + sourceNode.loop = this._loop; + if (this._loop) { + sourceNode.loopStart = this.startTime; + if (this.endTime) { + sourceNode.loopEnd = this.endTime; + } + } + } + + private _registerEntityTransformListener() { + this.entity.transform._updateFlagManager.addListener(this._onTransformChanged); + } + + private _updatePosition() { + const { _pannerNode: panner } = this; + const { position } = this.entity.transform; + const { context } = AudioManager; + + panner.positionX.setValueAtTime(position.x, context.currentTime); + panner.positionY.setValueAtTime(position.y, context.currentTime); + panner.positionZ.setValueAtTime(position.z, context.currentTime); + } + + private _updateOrientation() { + const { _pannerNode: panner } = this; + const { worldForward } = this.entity.transform; + const { context } = AudioManager; + + panner.orientationX.setValueAtTime(worldForward.x, context.currentTime); + panner.orientationY.setValueAtTime(worldForward.y, context.currentTime); + panner.orientationZ.setValueAtTime(worldForward.z, context.currentTime); + } +} diff --git a/packages/core/src/audio/index.ts b/packages/core/src/audio/index.ts new file mode 100644 index 0000000000..b4c963a182 --- /dev/null +++ b/packages/core/src/audio/index.ts @@ -0,0 +1,5 @@ +export { AudioManager } from "./AudioManager"; +export { AudioClip } from "./AudioClip"; +export { AudioListener } from "./AudioListener"; +export { AudioSource } from "./AudioSource"; +export { PositionalAudioSource } from "./PositionalAudioSource"; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 35f051cbc8..00abce26f5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -55,6 +55,7 @@ export * from "./clone/CloneManager"; export * from "./renderingHardwareInterface/index"; export * from "./physics/index"; export * from "./Utils"; +export * from "./audio"; // Export for CanvasRenderer plugin. export { Basic2DBatcher } from "./RenderPipeline/Basic2DBatcher"; diff --git a/packages/loader/src/AudioLoader.ts b/packages/loader/src/AudioLoader.ts new file mode 100644 index 0000000000..df048c5fa5 --- /dev/null +++ b/packages/loader/src/AudioLoader.ts @@ -0,0 +1,19 @@ +import { resourceLoader, Loader, AssetPromise, AssetType, LoadItem, AudioManager } from "@galacean/engine-core"; + +@resourceLoader(AssetType.Audio, ["mp3", "ogg", "wav"], false) +class AudioLoader extends Loader { + load(item: LoadItem): AssetPromise { + return new AssetPromise((resolve, reject) => { + this.request(item.url, { type: "arraybuffer" }).then((arrayBuffer) => { + AudioManager.context + .decodeAudioData(arrayBuffer) + .then((result) => { + resolve(result); + }) + .catch((e) => { + reject(e); + }); + }); + }); + } +} diff --git a/packages/loader/src/index.ts b/packages/loader/src/index.ts index 5e593ad254..7628188f94 100644 --- a/packages/loader/src/index.ts +++ b/packages/loader/src/index.ts @@ -15,6 +15,7 @@ import "./SpriteLoader"; import "./Texture2DLoader"; import "./TextureCubeLoader"; import "./AnimationClipLoader"; +import "./AudioLoader"; export { parseSingleKTX } from "./compressed-texture"; export type { GLTFParams } from "./GLTFLoader";