Skip to content

Commit

Permalink
AudioUtils and Waveform Seekbar
Browse files Browse the repository at this point in the history 
- Created AudioUtils for processing audio messages
- Created Waveform Seekbar, for visualizing a FloatArray
- Time limit of about 5 seconds, else shows regular seekbar
- Also made mediaPlayer smoother
- Fixed time discontinuity bug when playing voice messages, but only on API 28 or higher
Signed-off-by: Julius Linus <julius.linus@nextcloud.com>
  • Loading branch information
@rapterjet2004
rapterjet2004 committed Aug 1, 2023
1 parent 1f6ced7 commit 2430f72
Show file tree
Hide file tree
Showing 10 changed files with 468 additions and 70 deletions.
14 changes: 10 additions & 4 deletions app/src/main/java/com/nextcloud/talk/adapters/messages/IncomingVoiceMessageViewHolder.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,8 @@ class IncomingVoiceMessageViewHolder(incomingView: View, payload: Any) :
setParentMessageDataOnMessageItem(message)

updateDownloadState(message)
binding.seekbar.max = message.voiceMessageDuration - 1
viewThemeUtils.platform.themeHorizontalSeekBar(binding.seekbar)
binding.seekbar.max = message.voiceMessageDuration * ONE_SEC
viewThemeUtils.talk.themeWaveFormSeekBar(binding.seekbar)
viewThemeUtils.platform.colorCircularProgressBar(binding.progressBar, ColorRole.ON_SURFACE_VARIANT)

if (message.isPlayingVoiceMessage) {
Expand All @@ -115,7 +115,7 @@ class IncomingVoiceMessageViewHolder(incomingView: View, payload: Any) :
val t = message.voiceMessagePlayedSeconds.toLong()
binding.voiceMessageDuration.text = android.text.format.DateUtils.formatElapsedTime(d - t)
binding.voiceMessageDuration.visibility = View.VISIBLE
binding.seekbar.setProgress(message.voiceMessagePlayedSeconds, true)
binding.seekbar.progress = message.voiceMessageSeekbarProgress
} else {
binding.playPauseBtn.visibility = View.VISIBLE
binding.playPauseBtn.icon = ContextCompat.getDrawable(
Expand All @@ -127,6 +127,11 @@ class IncomingVoiceMessageViewHolder(incomingView: View, payload: Any) :
if (message.isDownloadingVoiceMessage) {
showVoiceMessageLoading()
} else {
if (message.voiceMessageFloatArray == null || message.voiceMessageFloatArray!!.isEmpty()) {
binding.seekbar.setWaveData(FloatArray(0))
} else {
binding.seekbar.setWaveData(message.voiceMessageFloatArray!!)
}
binding.progressBar.visibility = View.GONE
}

Expand All @@ -139,7 +144,7 @@ class IncomingVoiceMessageViewHolder(incomingView: View, payload: Any) :
binding.seekbar.progress = SEEKBAR_START
message.resetVoiceMessage = false
message.voiceMessagePlayedSeconds = 0
binding.voiceMessageDuration.visibility = View.GONE
binding.voiceMessageDuration.visibility = View.INVISIBLE
}

binding.seekbar.setOnSeekBarChangeListener(object : SeekBar.OnSeekBarChangeListener {
Expand Down Expand Up @@ -330,5 +335,6 @@ class IncomingVoiceMessageViewHolder(incomingView: View, payload: Any) :
companion object {
private const val TAG = "VoiceInMessageView"
private const val SEEKBAR_START: Int = 0
private const val ONE_SEC: Int = 1000
}
}
14 changes: 10 additions & 4 deletions app/src/main/java/com/nextcloud/talk/adapters/messages/OutcomingVoiceMessageViewHolder.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ class OutcomingVoiceMessageViewHolder(outcomingView: View) :
setParentMessageDataOnMessageItem(message)

updateDownloadState(message)
binding.seekbar.max = message.voiceMessageDuration - 1
viewThemeUtils.platform.themeHorizontalSeekBar(binding.seekbar)
binding.seekbar.max = message.voiceMessageDuration * ONE_SEC
viewThemeUtils.talk.themeWaveFormSeekBar(binding.seekbar)
viewThemeUtils.platform.colorCircularProgressBar(binding.progressBar, ColorRole.ON_SURFACE_VARIANT)

handleIsPlayingVoiceMessageState(message)
Expand Down Expand Up @@ -176,7 +176,7 @@ class OutcomingVoiceMessageViewHolder(outcomingView: View) :
)
binding.seekbar.progress = SEEKBAR_START
message.voiceMessagePlayedSeconds = 0
binding.voiceMessageDuration.visibility = View.GONE
binding.voiceMessageDuration.visibility = View.INVISIBLE
message.resetVoiceMessage = false
}
}
Expand All @@ -185,6 +185,11 @@ class OutcomingVoiceMessageViewHolder(outcomingView: View) :
if (message.isDownloadingVoiceMessage) {
showVoiceMessageLoading()
} else {
if (message.voiceMessageFloatArray == null || message.voiceMessageFloatArray!!.isEmpty()) {
binding.seekbar.setWaveData(FloatArray(0))
} else {
binding.seekbar.setWaveData(message.voiceMessageFloatArray!!)
}
binding.progressBar.visibility = View.GONE
}
}
Expand All @@ -201,7 +206,7 @@ class OutcomingVoiceMessageViewHolder(outcomingView: View) :
val t = message.voiceMessagePlayedSeconds.toLong()
binding.voiceMessageDuration.text = android.text.format.DateUtils.formatElapsedTime(d - t)
binding.voiceMessageDuration.visibility = View.VISIBLE
binding.seekbar.setProgress(message.voiceMessagePlayedSeconds, true)
binding.seekbar.progress = message.voiceMessageSeekbarProgress
} else {
binding.playPauseBtn.visibility = View.VISIBLE
binding.playPauseBtn.icon = ContextCompat.getDrawable(
Expand Down Expand Up @@ -313,5 +318,6 @@ class OutcomingVoiceMessageViewHolder(outcomingView: View) :
companion object {
private const val TAG = "VoiceOutMessageView"
private const val SEEKBAR_START: Int = 0
private const val ONE_SEC: Int = 1000
}
}
76 changes: 63 additions & 13 deletions app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ import com.nextcloud.talk.ui.dialog.ShowReactionsDialog
import com.nextcloud.talk.ui.recyclerview.MessageSwipeActions
import com.nextcloud.talk.ui.recyclerview.MessageSwipeCallback
import com.nextcloud.talk.utils.ApiUtils
import com.nextcloud.talk.utils.AudioUtils
import com.nextcloud.talk.utils.ContactUtils
import com.nextcloud.talk.utils.ConversationUtils
import com.nextcloud.talk.utils.DateConstants
Expand Down Expand Up @@ -232,6 +233,10 @@ import io.reactivex.Observer
import io.reactivex.android.schedulers.AndroidSchedulers
import io.reactivex.disposables.Disposable
import io.reactivex.schedulers.Schedulers
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.greenrobot.eventbus.Subscribe
import org.greenrobot.eventbus.ThreadMode
import retrofit2.HttpException
Expand Down Expand Up @@ -343,6 +348,11 @@ class ChatActivity :
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
)

// messy workaround for a mediaPlayer bug, don't delete
private var lastRecordMediaPosition: Int = 0
private var lastRecordedSeeked: Boolean = false

private lateinit var participantPermissions: ParticipantPermissions

private var videoURI: Uri? = null
Expand Down Expand Up @@ -861,21 +871,45 @@ class ChatActivity :
adapter?.setOnMessageViewLongClickListener { view, message -> onMessageViewLongClick(view, message) }
adapter?.registerViewClickListener(
R.id.playPauseBtn
) { view, message ->
) { _, message ->
val filename = message.selectedIndividualHashMap!!["name"]
val file = File(context.cacheDir, filename!!)
if (file.exists()) {
if (message.isPlayingVoiceMessage) {
pausePlayback(message)
} else {
startPlayback(message)
setUpWaveform(message)
}
} else {
Log.d(TAG, "Downloaded to cache")
downloadFileToCache(message)
}
}
}

private fun setUpWaveform(message: ChatMessage) {
val filename = message.selectedIndividualHashMap!!["name"]
val file = File(context.cacheDir, filename!!)
if (file.exists() && message.voiceMessageFloatArray == null) {
message.isDownloadingVoiceMessage = true
adapter?.update(message)
CoroutineScope(Dispatchers.Default).launch {
val bars = if (message.actorDisplayName == conversationUser?.displayName) {
NUM_BARS_OUTCOMING
} else {
NUM_BARS_INCOMING
}
val r = AudioUtils.audioFileToFloatArray(file, bars)
message.voiceMessageFloatArray = r
withContext(Dispatchers.Main) {
startPlayback(message)
}
}
} else {
startPlayback(message)
}
}

private fun initMessageHolders(): MessageHolders {
val messageHolders = MessageHolders()
val profileBottomSheet = ProfileBottomSheet(ncApi, conversationUser!!)
Expand Down Expand Up @@ -1215,7 +1249,6 @@ class ChatActivity :
setDataSource(currentVoiceRecordFile)
prepare()
setOnPreparedListener {
Log.d(TAG, "Julius the duration is ${it.duration}")
binding.messageInputView.seekBar.progress = 0
binding.messageInputView.seekBar.max = it.duration
voicePreviewObjectAnimator = ObjectAnimator.ofInt(
Expand Down Expand Up @@ -1742,6 +1775,7 @@ class ChatActivity :
mediaPlayer?.let {
if (!it.isPlaying) {
it.start()
Log.d(TAG, "MediaPlayer has Started")
}

mediaPlayerHandler = Handler()
Expand All @@ -1751,17 +1785,20 @@ class ChatActivity :
if (message.isPlayingVoiceMessage) {
val pos = mediaPlayer!!.currentPosition / VOICE_MESSAGE_SEEKBAR_BASE
if (pos < (mediaPlayer!!.duration / VOICE_MESSAGE_SEEKBAR_BASE)) {
lastRecordMediaPosition = mediaPlayer!!.currentPosition
message.voiceMessagePlayedSeconds = pos
message.voiceMessageSeekbarProgress = mediaPlayer!!.currentPosition
adapter?.update(message)
} else {
message.resetVoiceMessage = true
message.voiceMessagePlayedSeconds = 0
message.voiceMessageSeekbarProgress = 0
adapter?.update(message)
stopMediaPlayer(message)
}
}
}
mediaPlayerHandler.postDelayed(this, SECOND)
mediaPlayerHandler.postDelayed(this, 15)
}
})

Expand All @@ -1774,6 +1811,7 @@ class ChatActivity :
private fun pausePlayback(message: ChatMessage) {
if (mediaPlayer!!.isPlaying) {
mediaPlayer!!.pause()
Log.d(TAG, "MediaPlayer is paused")
}

message.isPlayingVoiceMessage = false
Expand All @@ -1794,13 +1832,22 @@ class ChatActivity :
mediaPlayer = MediaPlayer().apply {
setDataSource(absolutePath)
prepare()
}

currentlyPlayedVoiceMessage = message
message.voiceMessageDuration = mediaPlayer!!.duration / VOICE_MESSAGE_SEEKBAR_BASE

mediaPlayer!!.setOnCompletionListener {
stopMediaPlayer(message)
setOnPreparedListener {
currentlyPlayedVoiceMessage = message
message.voiceMessageDuration = mediaPlayer!!.duration / VOICE_MESSAGE_SEEKBAR_BASE
lastRecordedSeeked = false
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
setOnMediaTimeDiscontinuityListener { mp, _ ->
if (lastRecordMediaPosition > ONE_SECOND_IN_MILLIS && !lastRecordedSeeked) {
mp.seekTo(lastRecordMediaPosition)
lastRecordedSeeked = true
}
}
}
setOnCompletionListener {
stopMediaPlayer(message)
}
}
} catch (e: Exception) {
Log.e(TAG, "failed to initialize mediaPlayer", e)
Expand Down Expand Up @@ -1836,7 +1883,7 @@ class ChatActivity :
override fun updateMediaPlayerProgressBySlider(messageWithSlidedProgress: ChatMessage, progress: Int) {
if (mediaPlayer != null) {
if (messageWithSlidedProgress == currentlyPlayedVoiceMessage) {
mediaPlayer!!.seekTo(progress * VOICE_MESSAGE_SEEKBAR_BASE)
mediaPlayer!!.seekTo(progress)
}
}
}
Expand Down Expand Up @@ -1894,7 +1941,8 @@ class ChatActivity :
WorkManager.getInstance(context).getWorkInfoByIdLiveData(downloadWorker.id)
.observeForever { workInfo: WorkInfo ->
if (workInfo.state == WorkInfo.State.SUCCEEDED) {
startPlayback(message)
setUpWaveform(message)
// startPlayback(message)
}
}
}
Expand Down Expand Up @@ -4227,5 +4275,7 @@ class ChatActivity :
private const val TYPING_INTERVAL_TO_SEND_NEXT_TYPING_MESSAGE = 1000L
private const val TYPING_STARTED_SIGNALING_MESSAGE_TYPE = "startedTyping"
private const val TYPING_STOPPED_SIGNALING_MESSAGE_TYPE = "stoppedTyping"
private const val NUM_BARS_OUTCOMING: Int = 38
private const val NUM_BARS_INCOMING: Int = 50
}
}
20 changes: 17 additions & 3 deletions app/src/main/java/com/nextcloud/talk/models/json/chat/ChatMessage.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ import com.stfalcon.chatkit.commons.models.IUser
import com.stfalcon.chatkit.commons.models.MessageContentType
import kotlinx.parcelize.Parcelize
import java.security.MessageDigest
import java.util.Arrays
import java.util.Date

@Parcelize
Expand Down Expand Up @@ -132,15 +131,19 @@ data class ChatMessage(

var voiceMessagePlayedSeconds: Int = 0,

var voiceMessageDownloadProgress: Int = 0
var voiceMessageDownloadProgress: Int = 0,

var voiceMessageSeekbarProgress: Int = 0,

var voiceMessageFloatArray: FloatArray? = null

) : Parcelable, MessageContentType, MessageContentType.Image {

var extractedUrlToPreview: String? = null

// messageTypesToIgnore is weird. must be deleted by refactoring!!!
@JsonIgnore
var messageTypesToIgnore = Arrays.asList(
var messageTypesToIgnore = listOf(
MessageType.REGULAR_TEXT_MESSAGE,
MessageType.SYSTEM_MESSAGE,
MessageType.SINGLE_LINK_VIDEO_MESSAGE,
Expand Down Expand Up @@ -417,6 +420,17 @@ data class ChatMessage(
return map != null && MessageDigest.isEqual(map[key]!!.toByteArray(), searchTerm.toByteArray())
}

// needed a equals and hashcode function to fix detekt errors
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
return false
}

override fun hashCode(): Int {
return 0
}

val isVoiceMessage: Boolean
get() = "voice-message" == messageType
val isCommandMessage: Boolean
Expand Down
Loading

0 comments on commit 2430f72

Please sign in to comment.