sending cli command via connection interrupted by server reboot - always failed

[Roman]

i found the connection stay "open" even server rebooted (and from this moment will get ssh session closed exception) see the scenarios:

if i send 2 cli commands to server A
scenario 1:
1. connect to ssh
2. command 1
3. reboot A
4. command 2 (command got exception: SSH channel is closed. (The connection is being shutdown))

if i disconnecting after firs command completed , the second command succeeded
scenario 2:
1. connect to ssh
2. command 1
3. disconnect ssh
3. reboot A
4. command 2 (succeeded)

in my case i need the first scenario , how can i fix the first scenario ?

[Eran Cohanim]

Use the "rebootStation" method I created. this method reconnect to ssh server after reboot :)

[idans]

Hi Eran,
can you attach the function ?

[Roman]

this is the code (but this not fix this issue)
/*
* Reboot the work station and reconnect.
*
* @param timeToWaitBeforeReconnect
* - the time to wait for the machine to gracefully stop all
* services
* @param reconectTimeOut
* - int the number of seconds after which the device prompt must
* return. if fail to reconnect after this amount of seconds,
* throw exception
* @throws Exception
/
public void rebootStation(int timeToWaitBeforeReconnect, int reconectTimeOut) throws Exception {
StationCliCommand cliCommand = new StationCliCommand("reboot");
cliCommand.setSilent(isCommandSilent());
cliCommand.setIgnoreErrors(true);
conn.handleCliCommand("Reboot", cliCommand);
// sendAnyCommand("Reboot", cliCommand);

    this.waitForDeviceToReload(timeToWaitBeforeReconnect, reconectTimeOut);
    report.report("Machine reconnected - sleep 10 seconds for extra security");
    Thread.sleep(10000);
}

/**
 * Waits for device to reload and regain its prompt. This utility makes sure
 * device will not be connected before the first timeout . After that the
 * device will be checked for prompt return during the time out. Periodic
 * reconnect tries are done every 10 seconds
 * 
 * @param timeToWaitBeforeRconnect
 *            the time to wait for the machine to gracefully stop all
 *            services
 * @param timeout
 *            int the number of seconds after which the device prompt must
 *            return
 * @throws Exception
 */
public void waitForDeviceToReload(long timeToWaitBeforeRconnect, long timeout) throws Exception {
    long startTime = java.lang.System.currentTimeMillis();
    // initial time to wait before starting to handle the re-connection (let
    // the machine gracefully close all services
    report.report(getName() + " waiting for device to gracefully close all services. timeout: "
            + timeToWaitBeforeRconnect / 1000 + " seconds");
    Thread.sleep(timeToWaitBeforeRconnect);
    report.report(getName() + " waiting for device to reload. timeout: " + timeout / 1000 + " seconds");
    while (true) {
        // if (timeout > 0 && java.lang.System.currentTimeMillis() -
        // startTime > timeout) {
        // throw new Exception(getName() +
        // " waitForDeviceReload fail, timeout: " + timeout);
        // }
        Thread.sleep(10000);
        /* debug: */report.report(" waiting for device to reload. Time elapsed "
                + ((java.lang.System.currentTimeMillis() - startTime) / 1000) + " sec");
        conn.close();// Just In Case ...
        try {
            conn.connect();// re-connect
            conn.setClose(false);
            return;
        } catch (Exception e) {
            report.report("Caught exception while trying to re-connect", StringUtils.getStackTrace(e), false);
        }
    }
}

[idans]

This won't work because conn.connect() just trying 3 times... (what if your sleep time isn't enough for the connection protocol to go up ?)

Another (And this is the main reason !) , I think you need to use the conn.disconnect() function to be able to use connect again.

Here is the code i wrote and it works (a little dirty solution but its working)

public void rebootAndReconnect() throws Exception {

    report.report("Waiting for Host to reboot..");
    Thread waitForReboot = new Thread(new PingMonitor(this.getHost(),                 true), "WaitingForReboot");
    waitForReboot.start();

    CliCommand cliCommand = new CliCommand("reboot");
    cliCommand.setPromptString("the system is going down for reboot NOW!");
    cliCommand.setIgnoreErrors(true);
    getCliSession(false).handleCliCommand("Rebooting Node and reconneting", cliCommand);
    waitForReboot.join();
    int connectRetries = getCliApplication().conn.cli.getConnectRetries();
    getCliApplication().conn.cli.setConnectRetries(10000);
    getCliApplication().getConnectivityManager().getCli().disconnect(); 
    getCliApplication().getConnectivityManager().getCli().connect();
    getCliApplication().conn.cli.setConnectRetries(connectRetries);

}

[Roman]

i found there is a big difference between reboot sent from code or external reboot (my case is the second one)

see the my sendAnyCommand :

public void sendAnyCommand(String title, StationCliCommand cliCommand, boolean validateByPing, boolean pureOutput,
boolean disconnectAtEnd) throws Exception {
// report.report("reconnecting");
// conn.reconnect();

    if (!conn.isConnected()) {
        report.report("check ping to host: " + this.getHostName());
        if (validateByPing) {
            if (!this.isIpReachable(this.getHostName())) {
                report.report("Host: " + this.getHostName() + " not available for ping");
                throw new Exception("Host: " + this.getHostName() + " not available for ping");
            }
        }
        report.report("not connected , reconnect!!!");
        try {
            conn.connect();

        } catch (Exception e) {
            report.report("Can't connect to host: " + this.getHostName() + " " + e.getMessage() + " "
                    + e.getStackTrace());
            report.report("disconnectiong after commannd sent");
            conn.disconnect();
            throw new Exception("Can't connect to host: " + this.getHostName() + " " + e.getMessage() + " "
                    + e.getStackTrace());
        }
    }
    try {
        // cliCommand.setThrown(new Exception());

        conn.handleCliCommand(title, cliCommand);
    } catch (Exception e) {
        try {
            report.report("got exception:" + e.getMessage() + " after                               first try:", reportResult(Reporter.WARNING));
            report.report("reConnect & reSend command " + title);
            report.report("disconnecting ...");
            if (validateByPing) {
                if (!this.isIpReachable(this.getHostName())) {
                    report.report("Host: " + this.getHostName()                 +                                               " not available for ping");
                    throw new Exception("Host: " + this.getHostName() + " not available for ping");
                }
            }

            conn.disconnect();
            conn.cleanCliBuffer();
            conn.release();
            report.report("connecting ...");
            conn.connect();
            report.report("sending command " + title + "again ...");
            // cliCommand.setIgnoreErrors(true);

            conn.handleCliCommand(title, cliCommand);
        } catch (Exception e2) {
            String exeptionMessage = e2.getMessage();
            String expectedPrompts = "";
            if (exeptionMessage.contains("timeout")) {
                Prompt[] exepectedPromptArr = cliCommand.getPrompts();
                for (int i = 0; i < exepectedPromptArr.length; i++) {
                    expectedPrompts = expectedPrompts + " " + exepectedPromptArr[i].getPrompt();
                }
                exeptionMessage = " no one of those prompts: " + expectedPrompts + " not found \n"
                        + exeptionMessage;
            }
            report.report(exeptionMessage + " " + e.getMessage() + " " + e.getStackTrace(),
                    reportResult(Reporter.WARNING));
            throw new Exception(exeptionMessage + " " + e.getMessage() + " " + e.getStackTrace());
        }
    }
    if (pureOutput) {
        String commandAsString = cliCommand.getCommands()[cliCommand.getCommands().length - 1];
        String pureOutputString = this.getPureCommandOutput(commandAsString);
        try {
            this.setTestAgainstObject(pureOutputString);
            this.conn.setTestAgainstObject(pureOutputString);
        } catch (Exception e) {
            // TODO: handle exception
        }
    } else {
        this.setTestAgainstObject(conn.getTestAgainstObject());
    }
    if (disconnectAtEnd) {
        conn.disconnect();
        if (conn.isConnected())
            report.report("connection to:" + this.getHostName() + " still connnected ...");
        else
            report.report("connection to:" + this.getHostName() + " disconnected ...");
    }
}