001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.transport;
018
019import java.io.IOException;
020import java.util.Timer;
021import java.util.concurrent.BlockingQueue;
022import java.util.concurrent.LinkedBlockingQueue;
023import java.util.concurrent.RejectedExecutionException;
024import java.util.concurrent.RejectedExecutionHandler;
025import java.util.concurrent.SynchronousQueue;
026import java.util.concurrent.ThreadFactory;
027import java.util.concurrent.ThreadPoolExecutor;
028import java.util.concurrent.TimeUnit;
029import java.util.concurrent.atomic.AtomicBoolean;
030import java.util.concurrent.atomic.AtomicInteger;
031import java.util.concurrent.locks.ReentrantReadWriteLock;
032
033import org.apache.activemq.command.KeepAliveInfo;
034import org.apache.activemq.command.WireFormatInfo;
035import org.apache.activemq.thread.SchedulerTimerTask;
036import org.apache.activemq.util.ThreadPoolUtils;
037import org.apache.activemq.wireformat.WireFormat;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041/**
042 * Used to make sure that commands are arriving periodically from the peer of
043 * the transport.
044 */
045public abstract class AbstractInactivityMonitor extends TransportFilter {
046
047    private static final Logger LOG = LoggerFactory.getLogger(AbstractInactivityMonitor.class);
048
049    private static final long DEFAULT_CHECK_TIME_MILLS = 30000;
050
051    private static ThreadPoolExecutor ASYNC_TASKS;
052    private static int CHECKER_COUNTER;
053    private static Timer READ_CHECK_TIMER;
054    private static Timer WRITE_CHECK_TIMER;
055
056    private final AtomicBoolean monitorStarted = new AtomicBoolean(false);
057
058    private final AtomicBoolean commandSent = new AtomicBoolean(false);
059    private final AtomicBoolean inSend = new AtomicBoolean(false);
060    private final AtomicBoolean failed = new AtomicBoolean(false);
061
062    private final AtomicBoolean commandReceived = new AtomicBoolean(true);
063    private final AtomicBoolean inReceive = new AtomicBoolean(false);
064    private final AtomicInteger lastReceiveCounter = new AtomicInteger(0);
065
066    private final ReentrantReadWriteLock sendLock = new ReentrantReadWriteLock();
067
068    private SchedulerTimerTask connectCheckerTask;
069    private SchedulerTimerTask writeCheckerTask;
070    private SchedulerTimerTask readCheckerTask;
071
072    private long connectAttemptTimeout = DEFAULT_CHECK_TIME_MILLS;
073    private long readCheckTime = DEFAULT_CHECK_TIME_MILLS;
074    private long writeCheckTime = DEFAULT_CHECK_TIME_MILLS;
075    private long initialDelayTime = DEFAULT_CHECK_TIME_MILLS;
076    private boolean useKeepAlive = true;
077    private boolean keepAliveResponseRequired;
078
079    protected WireFormat wireFormat;
080
081    private final Runnable connectChecker = new Runnable() {
082
083        private final long startTime = System.currentTimeMillis();
084
085        @Override
086        public void run() {
087            long now = System.currentTimeMillis();
088
089            if ((now - startTime) >= connectAttemptTimeout && connectCheckerTask != null && !ASYNC_TASKS.isShutdown()) {
090                LOG.debug("No connection attempt made in time for {}! Throwing InactivityIOException.", AbstractInactivityMonitor.this.toString());
091                try {
092                    ASYNC_TASKS.execute(new Runnable() {
093                        @Override
094                        public void run() {
095                            onException(new InactivityIOException(
096                                "Channel was inactive (no connection attempt made) for too (>" + (connectAttemptTimeout) + ") long: " + next.getRemoteAddress()));
097                        }
098                    });
099                } catch (RejectedExecutionException ex) {
100                    if (!ASYNC_TASKS.isShutdown()) {
101                        LOG.error("Async connection timeout task was rejected from the executor: ", ex);
102                        throw ex;
103                    }
104                }
105            }
106        }
107    };
108
109    private final Runnable readChecker = new Runnable() {
110        long lastRunTime;
111
112        @Override
113        public void run() {
114            long now = System.currentTimeMillis();
115            long elapsed = (now - lastRunTime);
116
117            if (lastRunTime != 0) {
118                LOG.debug("{}ms elapsed since last read check.", elapsed);
119            }
120
121            // Perhaps the timer executed a read check late.. and then executes
122            // the next read check on time which causes the time elapsed between
123            // read checks to be small..
124
125            // If less than 90% of the read check Time elapsed then abort this
126            // read check.
127            if (!allowReadCheck(elapsed)) {
128                LOG.debug("Aborting read check...Not enough time elapsed since last read check.");
129                return;
130            }
131
132            lastRunTime = now;
133            readCheck();
134        }
135
136        @Override
137        public String toString() {
138            return "ReadChecker";
139        }
140    };
141
142    private boolean allowReadCheck(long elapsed) {
143        return elapsed > (readCheckTime * 9 / 10);
144    }
145
146    private final Runnable writeChecker = new Runnable() {
147        long lastRunTime;
148
149        @Override
150        public void run() {
151            long now = System.currentTimeMillis();
152            if (lastRunTime != 0) {
153                LOG.debug("{}: {}ms elapsed since last write check.", this, (now - lastRunTime));
154            }
155            lastRunTime = now;
156            writeCheck();
157        }
158
159        @Override
160        public String toString() {
161            return "WriteChecker";
162        }
163    };
164
165    public AbstractInactivityMonitor(Transport next, WireFormat wireFormat) {
166        super(next);
167        this.wireFormat = wireFormat;
168    }
169
170    @Override
171    public void start() throws Exception {
172        next.start();
173        startMonitorThreads();
174    }
175
176    @Override
177    public void stop() throws Exception {
178        stopMonitorThreads();
179        next.stop();
180    }
181
182    final void writeCheck() {
183        if (inSend.get()) {
184            LOG.trace("Send in progress. Skipping write check.");
185            return;
186        }
187
188        if (!commandSent.get() && useKeepAlive && monitorStarted.get() && !ASYNC_TASKS.isShutdown()) {
189            LOG.trace("{} no message sent since last write check, sending a KeepAliveInfo", this);
190
191            try {
192                ASYNC_TASKS.execute(new Runnable() {
193                    @Override
194                    public void run() {
195                        LOG.debug("Running {}", this);
196                        if (monitorStarted.get()) {
197                            try {
198                                // If we can't get the lock it means another
199                                // write beat us into the
200                                // send and we don't need to heart beat now.
201                                if (sendLock.writeLock().tryLock()) {
202                                    KeepAliveInfo info = new KeepAliveInfo();
203                                    info.setResponseRequired(keepAliveResponseRequired);
204                                    doOnewaySend(info);
205                                }
206                            } catch (IOException e) {
207                                onException(e);
208                            } finally {
209                                if (sendLock.writeLock().isHeldByCurrentThread()) {
210                                    sendLock.writeLock().unlock();
211                                }
212                            }
213                        }
214                    }
215
216                    @Override
217                    public String toString() {
218                        return "WriteCheck[" + getRemoteAddress() + "]";
219                    };
220                });
221            } catch (RejectedExecutionException ex) {
222                if (!ASYNC_TASKS.isShutdown()) {
223                    LOG.warn("Async write check was rejected from the executor: ", ex);
224                }
225            }
226        } else {
227            LOG.trace("{} message sent since last write check, resetting flag.", this);
228        }
229
230        commandSent.set(false);
231    }
232
233    final void readCheck() {
234        int currentCounter = next.getReceiveCounter();
235        int previousCounter = lastReceiveCounter.getAndSet(currentCounter);
236        if (inReceive.get() || currentCounter != previousCounter) {
237            LOG.trace("A receive is in progress, skipping read check.");
238            return;
239        }
240        if (!commandReceived.get() && monitorStarted.get() && !ASYNC_TASKS.isShutdown()) {
241            LOG.debug("No message received since last read check for {}. Throwing InactivityIOException.", this);
242
243            try {
244                ASYNC_TASKS.execute(new Runnable() {
245                    @Override
246                    public void run() {
247                        LOG.debug("Running {}", this);
248                        onException(new InactivityIOException("Channel was inactive for too (>" + readCheckTime + ") long: " + next.getRemoteAddress()));
249                    }
250
251                    @Override
252                    public String toString() {
253                        return "ReadCheck[" + getRemoteAddress() + "]";
254                    };
255                });
256            } catch (RejectedExecutionException ex) {
257                if (!ASYNC_TASKS.isShutdown()) {
258                    LOG.warn("Async read check was rejected from the executor: ", ex);
259                }
260            }
261        } else {
262            if (LOG.isTraceEnabled()) {
263                LOG.trace("Message received since last read check, resetting flag: {}", this);
264            }
265        }
266        commandReceived.set(false);
267    }
268
269    protected abstract void processInboundWireFormatInfo(WireFormatInfo info) throws IOException;
270
271    protected abstract void processOutboundWireFormatInfo(WireFormatInfo info) throws IOException;
272
273    @Override
274    public void onCommand(Object command) {
275        commandReceived.set(true);
276        inReceive.set(true);
277        try {
278            if (command.getClass() == KeepAliveInfo.class) {
279                KeepAliveInfo info = (KeepAliveInfo) command;
280                if (info.isResponseRequired()) {
281                    sendLock.readLock().lock();
282                    try {
283                        info.setResponseRequired(false);
284                        oneway(info);
285                    } catch (IOException e) {
286                        onException(e);
287                    } finally {
288                        sendLock.readLock().unlock();
289                    }
290                }
291            } else {
292                if (command.getClass() == WireFormatInfo.class) {
293                    synchronized (this) {
294                        try {
295                            processInboundWireFormatInfo((WireFormatInfo) command);
296                        } catch (IOException e) {
297                            onException(e);
298                        }
299                    }
300                }
301
302                transportListener.onCommand(command);
303            }
304        } finally {
305            inReceive.set(false);
306        }
307    }
308
309    @Override
310    public void oneway(Object o) throws IOException {
311        // To prevent the inactivity monitor from sending a message while we
312        // are performing a send we take a read lock. The inactivity monitor
313        // sends its Heart-beat commands under a write lock. This means that
314        // the MutexTransport is still responsible for synchronizing sends
315        sendLock.readLock().lock();
316        inSend.set(true);
317        try {
318            doOnewaySend(o);
319        } finally {
320            commandSent.set(true);
321            inSend.set(false);
322            sendLock.readLock().unlock();
323        }
324    }
325
326    // Must be called under lock, either read or write on sendLock.
327    private void doOnewaySend(Object command) throws IOException {
328        if (failed.get()) {
329            throw new InactivityIOException("Cannot send, channel has already failed: " + next.getRemoteAddress());
330        }
331        if (command.getClass() == WireFormatInfo.class) {
332            synchronized (this) {
333                processOutboundWireFormatInfo((WireFormatInfo) command);
334            }
335        }
336        next.oneway(command);
337    }
338
339    @Override
340    public void onException(IOException error) {
341        if (failed.compareAndSet(false, true)) {
342            stopMonitorThreads();
343            if (sendLock.writeLock().isHeldByCurrentThread()) {
344                sendLock.writeLock().unlock();
345            }
346            transportListener.onException(error);
347        }
348    }
349
350    public void setUseKeepAlive(boolean val) {
351        useKeepAlive = val;
352    }
353
354    public long getConnectAttemptTimeout() {
355        return connectAttemptTimeout;
356    }
357
358    public void setConnectAttemptTimeout(long connectionTimeout) {
359        this.connectAttemptTimeout = connectionTimeout;
360    }
361
362    public long getReadCheckTime() {
363        return readCheckTime;
364    }
365
366    public void setReadCheckTime(long readCheckTime) {
367        this.readCheckTime = readCheckTime;
368    }
369
370    public long getWriteCheckTime() {
371        return writeCheckTime;
372    }
373
374    public void setWriteCheckTime(long writeCheckTime) {
375        this.writeCheckTime = writeCheckTime;
376    }
377
378    public long getInitialDelayTime() {
379        return initialDelayTime;
380    }
381
382    public void setInitialDelayTime(long initialDelayTime) {
383        this.initialDelayTime = initialDelayTime;
384    }
385
386    public boolean isKeepAliveResponseRequired() {
387        return this.keepAliveResponseRequired;
388    }
389
390    public void setKeepAliveResponseRequired(boolean value) {
391        this.keepAliveResponseRequired = value;
392    }
393
394    public boolean isMonitorStarted() {
395        return this.monitorStarted.get();
396    }
397
398    abstract protected boolean configuredOk() throws IOException;
399
400    public synchronized void startConnectCheckTask() {
401        startConnectCheckTask(getConnectAttemptTimeout());
402    }
403
404    public synchronized void startConnectCheckTask(long connectionTimeout) {
405        if (connectionTimeout <= 0) {
406            return;
407        }
408
409        LOG.trace("Starting connection check task for: {}", this);
410
411        this.connectAttemptTimeout = connectionTimeout;
412
413        if (connectCheckerTask == null) {
414            connectCheckerTask = new SchedulerTimerTask(connectChecker);
415
416            synchronized (AbstractInactivityMonitor.class) {
417                if (CHECKER_COUNTER == 0) {
418                    if (ASYNC_TASKS == null || ASYNC_TASKS.isShutdown()) {
419                        ASYNC_TASKS = createExecutor();
420                    }
421                    if (READ_CHECK_TIMER == null) {
422                        READ_CHECK_TIMER = new Timer("ActiveMQ InactivityMonitor ReadCheckTimer", true);
423                    }
424                }
425                CHECKER_COUNTER++;
426                READ_CHECK_TIMER.schedule(connectCheckerTask, connectionTimeout);
427            }
428        }
429    }
430
431    public synchronized void stopConnectCheckTask() {
432        if (connectCheckerTask != null) {
433            LOG.trace("Stopping connection check task for: {}", this);
434            connectCheckerTask.cancel();
435            connectCheckerTask = null;
436
437            synchronized (AbstractInactivityMonitor.class) {
438                READ_CHECK_TIMER.purge();
439                CHECKER_COUNTER--;
440            }
441        }
442    }
443
444    protected synchronized void startMonitorThreads() throws IOException {
445        if (monitorStarted.get()) {
446            return;
447        }
448
449        if (!configuredOk()) {
450            return;
451        }
452
453        if (readCheckTime > 0) {
454            readCheckerTask = new SchedulerTimerTask(readChecker);
455        }
456
457        if (writeCheckTime > 0) {
458            writeCheckerTask = new SchedulerTimerTask(writeChecker);
459        }
460
461        if (writeCheckTime > 0 || readCheckTime > 0) {
462            monitorStarted.set(true);
463            synchronized (AbstractInactivityMonitor.class) {
464                if (ASYNC_TASKS == null || ASYNC_TASKS.isShutdown()) {
465                    ASYNC_TASKS = createExecutor();
466                }
467                if (READ_CHECK_TIMER == null) {
468                    READ_CHECK_TIMER = new Timer("ActiveMQ InactivityMonitor ReadCheckTimer", true);
469                }
470                if (WRITE_CHECK_TIMER == null) {
471                    WRITE_CHECK_TIMER = new Timer("ActiveMQ InactivityMonitor WriteCheckTimer", true);
472                }
473
474                CHECKER_COUNTER++;
475                if (readCheckTime > 0) {
476                    READ_CHECK_TIMER.schedule(readCheckerTask, initialDelayTime, readCheckTime);
477                }
478                if (writeCheckTime > 0) {
479                    WRITE_CHECK_TIMER.schedule(writeCheckerTask, initialDelayTime, writeCheckTime);
480                }
481            }
482        }
483    }
484
485    protected synchronized void stopMonitorThreads() {
486        stopConnectCheckTask();
487        if (monitorStarted.compareAndSet(true, false)) {
488            if (readCheckerTask != null) {
489                readCheckerTask.cancel();
490            }
491            if (writeCheckerTask != null) {
492                writeCheckerTask.cancel();
493            }
494
495            synchronized (AbstractInactivityMonitor.class) {
496                WRITE_CHECK_TIMER.purge();
497                READ_CHECK_TIMER.purge();
498                CHECKER_COUNTER--;
499                if (CHECKER_COUNTER == 0) {
500                    WRITE_CHECK_TIMER.cancel();
501                    READ_CHECK_TIMER.cancel();
502                    WRITE_CHECK_TIMER = null;
503                    READ_CHECK_TIMER = null;
504                    try {
505                        ThreadPoolUtils.shutdownGraceful(ASYNC_TASKS, 0);
506                    } finally {
507                        ASYNC_TASKS = null;
508                    }
509                }
510            }
511        }
512    }
513
514    private final ThreadFactory factory = new ThreadFactory() {
515        private long i = 0;
516        @Override
517        public Thread newThread(Runnable runnable) {
518            Thread thread = new Thread(runnable, "ActiveMQ InactivityMonitor Worker " + (i++));
519            thread.setDaemon(true);
520            return thread;
521        }
522    };
523
524    private ThreadPoolExecutor createExecutor() {
525        ThreadPoolExecutor exec = new ThreadPoolExecutor(getDefaultCorePoolSize(), getDefaultMaximumPoolSize(), getDefaultKeepAliveTime(),
526                TimeUnit.SECONDS, newWorkQueue(), factory, newRejectionHandler());
527        exec.allowCoreThreadTimeOut(true);
528        return exec;
529    }
530
531    private static int getDefaultKeepAliveTime() {
532        return Integer.getInteger("org.apache.activemq.transport.AbstractInactivityMonitor.keepAliveTime", 30);
533    }
534
535    private static int getDefaultCorePoolSize() {
536        return Integer.getInteger("org.apache.activemq.transport.AbstractInactivityMonitor.corePoolSize", 0);
537    }
538
539    private static int getDefaultMaximumPoolSize() {
540        return Integer.getInteger("org.apache.activemq.transport.AbstractInactivityMonitor.maximumPoolSize", Integer.MAX_VALUE);
541    }
542
543    private static int getDefaultWorkQueueCapacity() {
544        return Integer.getInteger("org.apache.activemq.transport.AbstractInactivityMonitor.workQueueCapacity", 0);
545    }
546
547    private static boolean canRejectWork() {
548        return Boolean.getBoolean("org.apache.activemq.transport.AbstractInactivityMonitor.rejectWork");
549    }
550
551    private BlockingQueue<Runnable> newWorkQueue() {
552        final int workQueueCapacity = getDefaultWorkQueueCapacity();
553        return workQueueCapacity > 0 ? new LinkedBlockingQueue<Runnable>(workQueueCapacity) : new SynchronousQueue<Runnable>();
554    }
555
556    private RejectedExecutionHandler newRejectionHandler() {
557        return canRejectWork() ? new ThreadPoolExecutor.AbortPolicy() : new ThreadPoolExecutor.CallerRunsPolicy();
558    }
559}