OPAL  Version 3.12.5
opalmixer.h
Go to the documentation of this file.
1 /*
2  * opalmixer.h
3  *
4  * OPAL media mixers
5  *
6  * Open Phone Abstraction Library (OPAL)
7  * Formally known as the Open H323 project.
8  *
9  * Copyright (C) 2007 Post Increment
10  *
11  * The contents of this file are subject to the Mozilla Public License
12  * Version 1.0 (the "License"); you may not use this file except in
13  * compliance with the License. You may obtain a copy of the License at
14  * http://www.mozilla.org/MPL/
15  *
16  * Software distributed under the License is distributed on an "AS IS"
17  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
18  * the License for the specific language governing rights and limitations
19  * under the License.
20  *
21  * The Original Code is Open Phone Abstraction Library.
22  *
23  * The Initial Developer of the Original Code is Post Increment
24  *
25  * Contributor(s): Craig Southeren (craigs@postincrement.com)
26  * Robert Jongbloed (robertj@voxlucida.com.au)
27  *
28  * $Revision: 29867 $
29  * $Author: rjongbloed $
30  * $Date: 2013-06-02 22:06:44 -0500 (Sun, 02 Jun 2013) $
31  */
32 
33 
34 #ifndef OPAL_OPAL_OPALMIXER_H
35 #define OPAL_OPAL_OPALMIXER_H
36 
37 #ifndef _PTLIB_H
38 #include <ptlib.h>
39 #endif
40 
41 #include <opal/buildopts.h>
42 
43 #if OPAL_HAS_MIXER
44 
45 #include <queue>
46 
47 #include <ep/localep.h>
48 #include <codec/vidcodec.h>
49 
50 
51 class RTP_DataFrame;
52 class OpalJitterBuffer;
54 
55 
56 //#define OPAL_MIXER_AUDIO_DEBUG 1
57 
58 
59 #define OPAL_OPT_LISTEN_ONLY "Listen-Only"
60 #define OPAL_OPT_CONF_OWNER "Conference-Owner"
61 
62 
64 
74 class OpalBaseMixer : public PObject
75 {
76  public:
78  bool pushThread,
79  unsigned periodMS,
80  unsigned periodTS
81  );
82 
83  virtual ~OpalBaseMixer();
84 
85  typedef PString Key_T;
86 
89  virtual bool AddStream(
90  const Key_T & key
91  );
92 
95  virtual void RemoveStream(
96  const Key_T & key
97  );
98 
101  virtual void RemoveAllStreams();
102 
108  virtual bool WriteStream(
109  const Key_T & key,
110  const RTP_DataFrame & input
111  );
112 
122  virtual RTP_DataFrame * ReadMixed();
123  virtual bool ReadMixed(RTP_DataFrame & mixed);
124 
135  virtual bool OnMixed(
136  RTP_DataFrame * & mixed
137  );
138 
142  void StartPushThread();
143 
148  void StopPushThread(bool lock = true);
149 
152  unsigned GetPeriodTS() const { return m_periodTS; }
153 
154  protected:
155  struct Stream : public PObject {
156  virtual ~Stream() { }
157  virtual void QueuePacket(const RTP_DataFrame & rtp) = 0;
158  std::queue<RTP_DataFrame> m_queue;
159  };
160  typedef std::map<Key_T, Stream *> StreamMap_T;
161 
162  virtual Stream * CreateStream() = 0;
163  virtual bool MixStreams(RTP_DataFrame & frame) = 0;
164  virtual size_t GetOutputSize() const = 0;
165 
166  virtual bool OnPush();
167  void PushThreadMain();
168 
169  bool m_pushThread; // true if to use a thread to push data out
170  unsigned m_periodMS; // Mixing interval in milliseconds
171  unsigned m_periodTS; // Mixing interval in timestamp units
172 
173  StreamMap_T m_inputStreams; // Map of key to stream for input RTP frame queues
174  unsigned m_outputTimestamp; // RTP timestamp for output data
175  RTP_DataFrame * m_pushFrame; // Cached frame for pushing RTP
176  PThread * m_workerThread; // reader thread handle
177  bool m_threadRunning; // used to stop reader thread
178  PMutex m_mutex; // mutex for list of streams and thread handle
179 };
180 
182 
192 {
193  public:
195  bool stereo = false,
196  unsigned sampleRate = OpalMediaFormat::AudioClockRate,
197  bool pushThread = true,
198  unsigned period = 10
199  );
200 
202 
205  virtual void RemoveStream(
206  const Key_T & key
207  );
208 
211  virtual void RemoveAllStreams();
212 
215  bool IsStereo() const { return m_stereo; }
216 
219  unsigned GetSampleRate() const { return m_sampleRate; }
220 
227  bool SetSampleRate(
228  unsigned rate
229  );
230 
237  bool SetJitterBufferSize(
238  const Key_T & key,
239  const OpalJitterBuffer::Init & init
240  );
241 
242  protected:
243  struct AudioStream : public Stream
244  {
245  AudioStream(OpalAudioMixer & mixer);
246  ~AudioStream();
247 
248  virtual void QueuePacket(const RTP_DataFrame & rtp);
249  const short * GetAudioDataPtr();
250 
253  unsigned m_nextTimestamp;
254  PShortArray m_cacheSamples;
256  };
257 
258  virtual Stream * CreateStream();
259  virtual bool MixStreams(RTP_DataFrame & frame);
260  virtual size_t GetOutputSize() const;
261 
262  void PreMixStreams();
263  void MixStereo(RTP_DataFrame & frame);
264  void MixAdditive(RTP_DataFrame & frame, const short * audioToSubtract);
265 
266  protected:
267  bool m_stereo;
268  unsigned m_sampleRate;
269 
272  std::vector<int> m_mixedAudio;
273 };
274 
275 
277 
278 #if OPAL_VIDEO
279 
287 {
288  public:
289  enum Styles {
308  };
309 
311  Styles style,
312  unsigned width,
313  unsigned height,
314  unsigned rate = 15,
315  bool pushThread = true
316  );
317 
319 
322  unsigned GetFrameWidth() const { return m_width; }
323 
326  unsigned GetFrameHeight() const { return m_height; }
327 
330  unsigned GetFrameRate() const { return 1000/m_periodMS; }
331 
335  bool SetFrameRate(
336  unsigned rate // New frames per second.
337  );
338 
342  bool SetFrameSize(
343  unsigned width,
344  unsigned height
345  );
346 
347  protected:
348  struct VideoStream : public Stream
349  {
350  VideoStream(OpalVideoMixer & mixer);
351  virtual void QueuePacket(const RTP_DataFrame & rtp);
352  void InsertVideoFrame(unsigned x, unsigned y, unsigned w, unsigned h);
353 
355  };
356 
357  friend struct VideoStream;
358 
359  virtual Stream * CreateStream();
360  virtual bool MixStreams(RTP_DataFrame & frame);
361  virtual size_t GetOutputSize() const;
362 
363  protected:
365  unsigned m_width, m_height;
367 
368  PBYTEArray m_frameStore;
370 };
371 
372 #endif // OPAL_VIDEO
373 
374 
376 
377 
383 {
384  OpalMixerNodeInfo(const char * name = NULL)
385  : m_name(name)
386  , m_closeOnEmpty(false)
387  , m_listenOnly(false)
388  , m_sampleRate(OpalMediaFormat::AudioClockRate)
389 #if OPAL_VIDEO
390  , m_audioOnly(false)
391  , m_style(OpalVideoMixer::eGrid)
392  , m_width(PVideoFrameInfo::CIFWidth)
393  , m_height(PVideoFrameInfo::CIFHeight)
394  , m_rate(15)
395 #endif
396  , m_mediaPassThru(false)
397  { }
398 
399  virtual ~OpalMixerNodeInfo() { }
400 
401  virtual OpalMixerNodeInfo * Clone() const { return new OpalMixerNodeInfo(*this); }
402 
403  PString m_name;
406  unsigned m_sampleRate;
407 #if OPAL_VIDEO
408  bool m_audioOnly;
410  unsigned m_width;
411  unsigned m_height;
412  unsigned m_rate;
413 #endif
417  PString m_displayText;
418  PString m_subject;
419  PString m_notes;
420  PString m_keywords;
421 };
422 
423 
425 
426 class OpalMixerNode;
427 
428 
434 {
435  public:
441 
445  virtual ~OpalMixerNodeManager();
446 
449  virtual void ShutDown();
450 
456  virtual PBoolean GarbageCollection();
458 
467  virtual OpalMixerNode * CreateNode(
468  OpalMixerNodeInfo * info
469  );
470 
476  virtual PSafePtr<OpalMixerNode> AddNode(
477  OpalMixerNodeInfo * info
478  );
479 
482  void AddNode(OpalMixerNode * node);
483 
487  PSafePtr<OpalMixerNode> GetFirstNode(
488  PSafetyMode mode = PSafeReference
489  ) const { return PSafePtr<OpalMixerNode>(m_nodesByUID, mode); }
490 
494  virtual PSafePtr<OpalMixerNode> FindNode(
495  const PString & name,
496  PSafetyMode mode = PSafeReference
497  );
498 
503  virtual void RemoveNode(
504  OpalMixerNode & node
505  );
506 
509  void AddNodeName(
510  PString name,
511  OpalMixerNode * node
512  );
513 
516  void RemoveNodeName(
517  PString name
518  );
519 
523  void RemoveNodeNames(
524  const PStringSet & names
525  );
526 
529  virtual PString CreateInternalURI(
530  const PGloballyUniqueID & guid
531  );
532 
537  virtual void OnNodeStatusChanged(
538  const OpalMixerNode & node,
540  );
541 
543  OpalManager & GetManager() const { return m_manager; }
545 
546  protected:
548 
549  PSafeDictionary<PGloballyUniqueID, OpalMixerNode> m_nodesByUID;
550  PSafeDictionary<PString, OpalMixerNode> m_nodesByName;
551 };
552 
553 
555 
561 {
563  public:
569  OpalManager & manager,
570  const char * prefix
571  );
572 
576 
581  virtual void ShutDown();
583 
596  virtual OpalMediaFormatList GetMediaFormats() const;
597 
625  virtual PSafePtr<OpalConnection> MakeConnection(
626  OpalCall & call,
627  const PString & party,
628  void * userData = NULL,
629  unsigned options = 0,
630  OpalConnection::StringOptions * stringOptions = NULL
631  );
632 
648  virtual bool GetConferenceStates(
649  OpalConferenceStates & states,
650  const PString & name = PString::Empty()
651  ) const;
652 
657  virtual PBoolean GarbageCollection();
659 
668  PSafePtr<OpalMixerConnection> GetMixerConnectionWithLock(
669  const PString & token,
670  PSafetyMode mode = PSafeReadWrite
671  ) { return GetConnectionWithLockAs<OpalMixerConnection>(token, mode); }
672 
677  PSafePtr<OpalMixerNode> node,
678  OpalCall & call,
679  void * userData,
680  unsigned options,
681  OpalConnection::StringOptions * stringOptions
682  );
683 
688  const PString & name
689  );
691 
706  void SetAdHocNodeInfo(
707  const OpalMixerNodeInfo & info
708  );
709  void SetAdHocNodeInfo(
710  OpalMixerNodeInfo * info
711  );
712 
725 
743  void SetFactoryNodeInfo(
744  const OpalMixerNodeInfo & info
745  );
746  void SetFactoryNodeInfo(
747  OpalMixerNodeInfo * info
748  );
749 
754  virtual PString GetNewFactoryName();
755 
759 
760  protected:
761  virtual PString CreateInternalURI(const PGloballyUniqueID & guid);
762  virtual void OnNodeStatusChanged(const OpalMixerNode & node, OpalConferenceState::ChangeType change);
763 
766  PMutex m_infoMutex; // For above two fields
767  PAtomicInteger m_factoryIndex;
768 };
769 
770 
772 
776 {
778  public:
784  PSafePtr<OpalMixerNode> node,
785  OpalCall & call,
787  void * userData,
788  unsigned options = 0,
789  OpalConnection::StringOptions * stringOptions = NULL
790  );
791 
796 
816  virtual void OnReleased();
817 
824  virtual OpalMediaFormatList GetMediaFormats() const;
825 
841  const OpalMediaFormat & mediaFormat,
842  unsigned sessionID,
843  PBoolean isSource
844  );
845 
848  virtual void OnStartMediaPatch(
849  OpalMediaPatch & patch
850  );
851 
853  virtual void OnApplyStringOptions();
854 
861  virtual PBoolean SendUserInputString(
862  const PString & value
863  );
864 
881  virtual PBoolean SendUserInputTone(
882  char tone,
883  unsigned duration = 0
884  );
885 
898  virtual bool GetConferenceState(
899  OpalConferenceState * state
900  ) const;
902 
907  void SetListenOnly(
908  bool listenOnly
909  );
910 
913  bool GetListenOnly() const { return m_listenOnly; }
914 
917  PSafePtr<OpalMixerNode> GetNode() const { return m_node; }
919 
920  protected:
922  PSafePtr<OpalMixerNode> m_node;
924 };
925 
926 
931 {
933  public:
939  OpalConnection & conn,
940  const OpalMediaFormat & mediaFormat,
941  unsigned sessionID,
942  bool isSource,
943  PSafePtr<OpalMixerNode> node,
944  bool listenOnly
945  );
946 
951 
956  virtual PBoolean Open();
957 
963  virtual PBoolean WritePacket(
964  RTP_DataFrame & packet
965  );
966 
970  virtual PBoolean IsSynchronous() const;
971 
982  virtual PBoolean RequiresPatchThread() const;
984 
989  PSafePtr<OpalMixerNode> GetNode() { return m_node; }
991 
992  protected:
993  virtual void InternalClose();
994  virtual bool InternalSetJitterBuffer(const OpalJitterBuffer::Init & init) const;
995 
996  PSafePtr<OpalMixerNode> m_node;
998 #if OPAL_VIDEO
999  bool m_video;
1000 #endif
1001 };
1002 
1003 
1007 class OpalMixerNode : public PSafeObject
1008 {
1009  PCLASSINFO(OpalMixerNode, PSafeObject);
1010  public:
1015  OpalMixerNode(
1016  OpalMixerNodeManager & manager,
1017  OpalMixerNodeInfo * info
1018  );
1019 
1022  ~OpalMixerNode();
1023 
1028  void ShutDown();
1030 
1037  void PrintOn(
1038  ostream & strm
1039  ) const;
1041 
1046  virtual void AttachConnection(
1047  OpalConnection * connection
1048  );
1049 
1052  virtual void DetachConnection(
1053  OpalConnection * connection
1054  );
1055 
1058  virtual bool AttachStream(
1059  OpalMixerMediaStream * stream
1060  );
1061 
1064  virtual void DetachStream(
1065  OpalMixerMediaStream * stream
1066  );
1067 
1070  virtual void UseMediaPassThrough(
1071  unsigned sessionID,
1072  OpalConnection * connection = NULL
1073  );
1074 
1082  const OpalBaseMixer::Key_T & key,
1083  const OpalJitterBuffer::Init & init
1084  ) { return m_audioMixer.SetJitterBufferSize(key, init); }
1085 
1089  const OpalBaseMixer::Key_T & key,
1090  const RTP_DataFrame & input
1091  ) { return m_audioMixer.WriteStream(key, input); }
1092 
1093 #if OPAL_VIDEO
1094 
1097  const OpalBaseMixer::Key_T & key,
1098  const RTP_DataFrame & input
1099  ) { return m_videoMixer.WriteStream(key, input); }
1100 #endif // OPAL_VIDEO
1101 
1104  virtual void BroadcastUserInput(
1105  const OpalConnection * connection,
1106  const PString & value
1107  );
1108 
1121  virtual void GetConferenceState(
1122  OpalConferenceState & state
1123  ) const;
1125 
1130  const PGloballyUniqueID & GetGUID() const { return m_guid; }
1131 
1134  const PStringSet & GetNames() const { return m_names; }
1135 
1138  void AddName(
1139  const PString & name
1140  );
1141 
1144  void RemoveName(
1145  const PString & name
1146  );
1147 
1153  PINDEX GetConnectionCount() const { return m_connections.GetSize(); }
1154 
1157  template <class Subclass>
1158  PSafePtr<Subclass> GetFirstConnectionAs(
1159  PSafetyMode mode = PSafeReference
1160  ) const { return PSafePtr<Subclass>(m_connections, mode); }
1161 
1164  PSafePtr<OpalConnection> GetFirstConnection(
1165  PSafetyMode mode = PSafeReference
1166  ) const { return GetFirstConnectionAs<OpalConnection>(mode); }
1167 
1170  const OpalMixerNodeInfo & GetNodeInfo() { return *m_info; }
1171 
1174  const PTime & GetCreationTime() const { return m_creationTime; }
1175 
1183  const PString & connectionIdentifier
1184  ) { m_ownerConnection = connectionIdentifier; }
1186 
1187  protected:
1189  PGloballyUniqueID m_guid;
1190  PStringSet m_names;
1193  PAtomicBoolean m_shuttingDown;
1194 
1195  PSafeList<OpalConnection> m_connections;
1197 
1198  struct MediaMixer
1199  {
1200  MediaMixer();
1201  void CloseOne(const PSafePtr<OpalMixerMediaStream> & stream);
1202 
1203  PSafeList<OpalMixerMediaStream> m_outputStreams;
1204  };
1205 
1206  struct AudioMixer : public OpalAudioMixer, public MediaMixer
1207  {
1208  AudioMixer(const OpalMixerNodeInfo & info);
1209  ~AudioMixer();
1210 
1211  virtual bool OnPush();
1212 
1213  struct CachedAudio {
1214  CachedAudio();
1215  ~CachedAudio();
1220  };
1221  std::map<PString, CachedAudio> m_cache;
1222 
1223  void PushOne(
1224  PSafePtr<OpalMixerMediaStream> & stream,
1225  CachedAudio & cache,
1226  const short * audioToSubtract
1227  );
1228 #ifdef OPAL_MIXER_AUDIO_DEBUG
1229  class PAudioMixerDebug * m_audioDebug;
1230 #endif
1231  };
1233 
1234 #if OPAL_VIDEO
1235  struct VideoMixer : public OpalVideoMixer, public MediaMixer
1236  {
1237  VideoMixer(const OpalMixerNodeInfo & info);
1238  ~VideoMixer();
1239 
1240  virtual bool OnMixed(RTP_DataFrame * & output);
1241 
1242  PDictionary<PString, OpalTranscoder> m_transcoders;
1243  };
1245 #endif // OPAL_VIDEO
1246 };
1247 
1248 
1249 #endif // OPAL_HAS_MIXER
1250 
1251 #endif // OPAL_OPAL_OPAL_MIXER
1252 
1253