Fixed monitoring of the admin and debug ports when stopping

Signed-off-by: David Matějček <[email protected]>

Author: dmatej

nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java

@@ -382,7 +382,6 @@ public String getLogFilename() {
         throw new IllegalStateException("Call to getLogFilename() before it has been initialized!");
     }
 
-
     /**
      * @return null or a port number
      */

nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ChangeMasterPasswordCommand.java

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2025 Contributors to the Eclipse Foundation.
  * Copyright (c) 1997, 2018 Oracle and/or its affiliates. All rights reserved.
  *
  * This program and the accompanying materials are made available under the
@@ -92,21 +93,19 @@ protected int executeCommand() throws CommandException {
             if (nodeDir != null) {
                 command = CLICommand.getCommand(habitat, CHANGE_MASTER_PASSWORD_NODE);
                 return command.execute(argv);
-            } else {
-
-                // nodeDir is not specified and domainNameOrNodeName is not a domain.
-                // It could be a node
-                // We add defaultNodeDir parameter to args
-                ArrayList arguments = new ArrayList<String>(Arrays.asList(argv));
-                arguments.remove(argv.length - 1);
-                arguments.add("--nodedir");
-                arguments.add(getDefaultNodesDirs().getAbsolutePath());
-                arguments.add(domainNameOrNodeName);
-                String[] newargs = (String[]) arguments.toArray(new String[arguments.size()]);
-
-                command = CLICommand.getCommand(habitat, CHANGE_MASTER_PASSWORD_NODE);
-                return command.execute(newargs);
             }
+            // nodeDir is not specified and domainNameOrNodeName is not a domain.
+            // It could be a node
+            // We add defaultNodeDir parameter to args
+            ArrayList<String> arguments = new ArrayList<>(Arrays.asList(argv));
+            arguments.remove(argv.length - 1);
+            arguments.add("--nodedir");
+            arguments.add(getDefaultNodesDirs().getAbsolutePath());
+            arguments.add(domainNameOrNodeName);
+            String[] newargs = arguments.toArray(String[]::new);
+
+            command = CLICommand.getCommand(habitat, CHANGE_MASTER_PASSWORD_NODE);
+            return command.execute(newargs);
         } catch (IOException e) {
             throw new CommandException(e.getMessage(), e);
         }

nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ListDomainsCommand.java

@@ -94,7 +94,7 @@ private DomainInfo getStatus(String domainName) throws CommandException {
         setDomainName(domainName);
         initDomain();
         DomainInfo info = new DomainInfo();
-        info.adminAddr = getAdminAddress(getDomainXml(), "server");
+        info.adminAddr = getAdminAddresses(getDomainXml(), "server");
         HostAndPort reachableEndpoint = getReachableAdminAddress(getDomainXml());
         if (reachableEndpoint == null) {
             info.status = false;

nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java

@@ -33,10 +33,12 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.System.Logger;
+import java.lang.System.Logger.Level;
 import java.nio.file.Files;
 import java.time.Duration;
 import java.util.List;
-import java.util.logging.Level;
+import java.util.function.Supplier;
 
 import org.glassfish.api.ActionReport;
 import org.glassfish.api.ActionReport.ExitCode;
@@ -50,21 +52,23 @@
 import static com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.step;
 import static com.sun.enterprise.universal.process.ProcessUtils.loadPid;
 import static com.sun.enterprise.universal.process.ProcessUtils.waitForNewPid;
+import static com.sun.enterprise.universal.process.ProcessUtils.waitWhileIsAlive;
 import static com.sun.enterprise.util.SystemPropertyConstants.KEYSTORE_PASSWORD_DEFAULT;
 import static com.sun.enterprise.util.SystemPropertyConstants.MASTER_PASSWORD_ALIAS;
 import static com.sun.enterprise.util.SystemPropertyConstants.MASTER_PASSWORD_FILENAME;
 import static com.sun.enterprise.util.SystemPropertyConstants.MASTER_PASSWORD_PASSWORD;
 import static com.sun.enterprise.util.SystemPropertyConstants.TRUSTSTORE_FILENAME_DEFAULT;
-import static java.util.logging.Level.FINER;
-import static java.util.logging.Level.FINEST;
+import static java.lang.System.Logger.Level.DEBUG;
+import static java.lang.System.Logger.Level.INFO;
+import static java.lang.System.Logger.Level.TRACE;
 
 /**
  * A class that's supposed to capture all the behavior common to operation on a "local" server.
  *
  * @author Byron Nevins
  */
 public abstract class LocalServerCommand extends CLICommand {
-
+    private static final Logger LOG = System.getLogger(LocalServerCommand.class.getName());
     private static final LocalStringsImpl I18N = new LocalStringsImpl(LocalDomainCommand.class);
     private ServerDirs serverDirs;
 
@@ -105,20 +109,24 @@ protected final HostAndPort getReachableAdminAddress(File domainXml) throws Comm
 
     private HostAndPort findReachableAdminAddress(String userHost, Integer userPort, Boolean userSecure, File domainXml)
         throws CommandException {
+        LOG.log(DEBUG, "findReachableAdminAddress(userHost={0}, userPort={1}, userSecure={2}, domainXml={3})",
+            userHost, userPort, userSecure, domainXml);
         // Respect user provided values
         if (userHost != null && userPort != null && userSecure != null) {
             HostAndPort endpoint = new HostAndPort(userHost, userPort, userSecure);
             return ProcessUtils.isListening(endpoint) ? endpoint : null;
         }
         // default: DAS always has the name "server"
-        final List<HostAndPort> xml = getAdminAddress(domainXml, "server");
+        final List<HostAndPort> xml = getAdminAddresses(domainXml, "server");
         if (xml.isEmpty()) {
             String host = userHost == null ? DEFAULT_HOSTNAME : userHost;
             Integer port = userPort == null ? DEFAULT_ADMIN_PORT : userPort;
             boolean secure = userSecure == null ? false : userSecure;
             HostAndPort endpoint = new HostAndPort(host, port, secure);
+            LOG.log(INFO, "Checking candidate: " + endpoint);
             return ProcessUtils.isListening(endpoint) ? endpoint : null;
         }
+        LOG.log(TRACE, () -> "Checking candidates: " + xml);
         for (HostAndPort candidate : xml) {
             final String host = userHost == null ? candidate.getHost() : userHost;
             final int port = userPort == null ? candidate.getPort() : userPort;
@@ -136,9 +144,10 @@ private HostAndPort findReachableAdminAddress(String userHost, Integer userPort,
      * For remote access it uses command line arguments and defaults.
      *
      * @return list of HostAndPort objects with admin server addresses. Never null.
-     * @throws CommandException in case of parsing errors
+     * @throws IllegalStateException in case of parsing errors
      */
-    protected final List<HostAndPort> getAdminAddress(File domainXml, String serverName) throws CommandException {
+    protected final List<HostAndPort> getAdminAddresses(File domainXml, String serverName)
+        throws IllegalStateException {
         if (!isLocal()) {
             // We don't have any access to changes.
             return List.of(getUserProvidedAdminAddress());
@@ -147,11 +156,11 @@ protected final List<HostAndPort> getAdminAddress(File domainXml, String serverN
             MiniXmlParser parser = new MiniXmlParser(domainXml, serverName);
             List<HostAndPort> addrSet = parser.getAdminAddresses();
             if (addrSet.isEmpty()) {
-                throw new CommandException(I18N.get("NoAdminPort"));
+                throw new IllegalStateException(I18N.get("NoAdminPort"));
             }
             return addrSet;
         } catch (MiniXmlParserException ex) {
-            throw new CommandException(I18N.get("NoAdminPortEx", ex), ex);
+            throw new IllegalStateException(I18N.get("NoAdminPortEx", ex), ex);
         }
     }
 
@@ -178,7 +187,7 @@ protected final boolean isLocal() {
     protected final void setLocalPassword() {
         String pw = serverDirs == null ? null : serverDirs.getLocalPassword();
         programOpts.setPassword(pw == null ? null : pw.toCharArray(), LOCAL_PASSWORD);
-        logger.finer(ok(pw) ? "Using local password" : "Not using local password");
+        LOG.log(DEBUG, () -> ok(pw) ? "Using local password" : "Not using local password");
     }
 
     protected final void unsetLocalPassword() {
@@ -220,7 +229,7 @@ protected final String readFromMasterPasswordFile() {
             PasswordAdapter pw = new PasswordAdapter(mpf.getAbsolutePath(), MASTER_PASSWORD_PASSWORD.toCharArray());
             return pw.getPasswordForAlias(MASTER_PASSWORD_ALIAS);
         } catch (Exception e) {
-            logger.log(Level.WARNING, "A master password file reading error: " + e.toString(), e);
+            LOG.log(Level.WARNING, "A master password file reading error: " + e.toString(), e);
             return null;
         }
     }
@@ -230,15 +239,15 @@ protected final boolean verifyMasterPassword(String mpv) {
     }
 
     protected boolean loadAndVerifyKeystore(File jks, String mpv) {
-        logger.log(FINEST, "loading keystore: " + jks);
+        LOG.log(DEBUG, "loading keystore: " + jks);
         if (jks == null || mpv == null) {
             return false;
         }
         try {
             new KeyTool(jks, mpv.toCharArray()).loadKeyStore();
             return true;
         } catch (Exception e) {
-            logger.log(FINER, e.getMessage(), e);
+            LOG.log(DEBUG, e.getMessage(), e);
             return false;
         }
     }
@@ -268,7 +277,7 @@ protected final String getMasterPassword() throws CommandException {
                 masterPassword = retry(countOfRetries);
             }
         }
-        logger.log(FINER, "Time spent in master password extraction: {0} ms", (System.currentTimeMillis() - start));
+        LOG.log(DEBUG, "Time spent in master password extraction: " + (System.currentTimeMillis() - start) + " ms");
         return masterPassword;
     }
 
@@ -283,13 +292,13 @@ protected final boolean isThisServer(File ourDir, String directoryKey) {
         }
 
         ourDir = getUniquePath(ourDir);
-        logger.log(FINER, "Check if server is at location {0}", ourDir);
+        LOG.log(DEBUG, "Check if server is at location {0}", ourDir);
 
         try {
             RemoteCLICommand cmd = new RemoteCLICommand("__locations", programOpts, env);
             ActionReport report = cmd.executeAndReturnActionReport(new String[] { "__locations" });
             String theirDirPath = report.findProperty(directoryKey);
-            logger.log(FINER, "Remote server has root directory {0}", theirDirPath);
+            LOG.log(DEBUG, "Remote server has root directory {0}", theirDirPath);
 
             if (ok(theirDirPath)) {
                 File theirDir = getUniquePath(new File(theirDirPath));
@@ -315,14 +324,14 @@ protected final Long getServerPid() {
             if (report.getActionExitCode() == ExitCode.SUCCESS) {
                 long pidFromAdmin = Long.parseLong(report.findProperty("Pid"));
                 if (pidFromFile == null || !pidFromFile.equals(pidFromAdmin)) {
-                    logger.log(Level.SEVERE, "PID should be the same: PID from file = " + pidFromFile
+                    LOG.log(Level.ERROR, "PID should be the same: PID from file = " + pidFromFile
                         + ", while PID received from admin endpoint = " + pidFromAdmin);
                 }
                 return pidFromAdmin;
             }
             return null;
         } catch (Exception e) {
-            logger.log(Level.SEVERE, "The server PID could not be resolved, sending PID from file: " + pidFromFile, e);
+            LOG.log(Level.ERROR, "The server PID could not be resolved, sending PID from file: " + pidFromFile + ".", e);
             return pidFromFile;
         }
     }
@@ -331,35 +340,50 @@ protected final Long getServerPid() {
      * Waits until server stops
      *
      * @param pid
-     * @param oldAdminAddress
+     * @param adminAddress
      * @param timeout can be null
      * @throws CommandException if we time out.
      */
-    protected final void waitForStop(final Long pid, final HostAndPort oldAdminAddress, final Duration timeout)
+    protected final void waitForStop(final Long pid, final HostAndPort adminAddress, final Duration timeout)
         throws CommandException {
-        logger.log(Level.FINEST, "waitForStop(pid={0}, oldAdminAddress={1}, timeout={2})",
-            new Object[] {pid, oldAdminAddress, timeout});
+        LOG.log(DEBUG, "waitForStop(pid={0}, oldAdminAddress={1}, timeout={2})", pid, adminAddress, timeout);
 
         final boolean printDots = !programOpts.isTerse();
-        final boolean stopped = pid == null || ProcessUtils.waitWhileIsAlive(pid, timeout, printDots);
-        if (!stopped) {
-            throw new CommandException("Timed out waiting for the server to stop.");
+
+        final Duration portTimeout;
+        if (pid == null) {
+            portTimeout = timeout;
+        } else {
+            portTimeout = step("Waiting for the death of the process with pid " + pid, timeout,
+                () -> waitWhileIsAlive(pid, timeout, printDots));
+            if (ProcessUtils.isAlive(pid)) {
+                throw new CommandException("Timed out waiting for the server process to stop.");
+            }
+        }
+        if (adminAddress == null) {
+            return;
         }
+        LOG.log(INFO, "Waiting until admin endpoint {0} is free.", adminAddress);
+        final boolean stopped = ProcessUtils.waitWhileListening(adminAddress,
+            portTimeout == null ? Duration.ofHours(1L) : portTimeout, printDots);
+        if (stopped) {
+            return;
+        }
+        throw new CommandException("Timed out waiting for the server to stop.");
     }
 
     /**
      * Waits until server is running - with different pid
      *
      * @param oldPid
      * @param lifeSignCheck
-     * @param adminEndpoints
+     * @param adminEndpointsSupplier
      * @param timeout can be null
      * @throws CommandException if we time out.
      */
     protected final String waitForStart(final Long oldPid, final ServerLifeSignCheck lifeSignCheck,
-        final List<HostAndPort> adminEndpoints, final Duration timeout) throws CommandException {
-        logger.log(Level.FINEST, "waitForStart(oldPid={0}, adminEndpoints={1}, timeout={2})",
-            new Object[] {oldPid, adminEndpoints, timeout});
+        final Supplier<List<HostAndPort>> adminEndpointsSupplier, final Duration timeout) throws CommandException {
+        LOG.log(DEBUG, "waitForStart(oldPid={0}, adminEndpoints, timeout={1})", oldPid, timeout);
 
         final boolean printDots = !programOpts.isTerse();
         final File pidFile = getServerDirs().getPidFile();
@@ -378,7 +402,16 @@ protected final String waitForStart(final Long oldPid, final ServerLifeSignCheck
             throw new CommandException(reportPidFileIssue(pidFile));
         }
 
-        logger.log(Level.INFO, () -> "Waiting until start of " + lifeSignCheck.getServerTitleAndName() + " completes.");
+        try {
+            resetServerDirs();
+            setLocalPassword();
+        } catch (Exception e) {
+            LOG.log(Level.WARNING, "The endpoint is alive, but we failed to reset the local password.", e);
+        }
+
+        LOG.log(INFO, () -> "Waiting until start of " + lifeSignCheck.getServerTitleAndName() + " completes.");
+
+        final List<HostAndPort> adminEndpoints = adminEndpointsSupplier.get();
         final ServerLifeSignChecker checker = new ServerLifeSignChecker(lifeSignCheck, pidFile, adminEndpoints, printDots);
         final GlassFishProcess process = GlassFishProcess.of(pid);
         final ServerLifeSigns signs = checker.watchStartup(process, startTimeout);
@@ -424,14 +457,14 @@ private String loadRestartLog(final File logFile ) {
             return null;
         }
         try {
-            String report = "Found restart log file content: \n***********\n"
-                + Files.readString(serverDirs.getRestartLogFile().toPath()) + "\n**********\n";
+            String report = "Found restart log file content: \n##########\n"
+                + Files.readString(serverDirs.getRestartLogFile().toPath()) + "\n##########\n";
             // The log is there just to diagnose failed start phase of the restart.
             // We don't want to delete it if we cannot load it.
             logFile.delete();
             return report;
         } catch (IOException e) {
-            logger.log(Level.WARNING, "Failed to read the restart log file: " + serverDirs.getRestartLogFile(), e);
+            LOG.log(Level.WARNING, "Failed to read the restart log file: " + serverDirs.getRestartLogFile(), e);
             return null;
         }
     }
@@ -448,7 +481,7 @@ protected final long getUptime() throws CommandException {
             throw new CommandException(I18N.get("restart.dasNotRunning"));
         }
 
-        logger.log(FINER, "server uptime: {0}", up_ms);
+        LOG.log(DEBUG, () -> "Server uptime: " + up_ms + " ms");
         return up_ms;
     }
 
@@ -495,7 +528,7 @@ private File getJKS() {
         if (mp.canRead()) {
             return mp;
         }
-        logger.log(FINEST, "File does not exist or is not readable: {0}", mp);
+        LOG.log(DEBUG, "File does not exist or is not readable: {0}", mp);
         return null;
     }
 

nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalStrings.properties

@@ -115,7 +115,6 @@ Domain.badDomainDir=CLI301: There is no such domain directory: {0}
 Domain.noDomainXml=CLI304: Cannot find domain.xml.  It should be here: {0}
 
 ## stop-domain command
-StopDomain.WaitDASDeath=Waiting for the domain to stop
 StopDomain.DASNotDead=Timed out ({0} seconds) waiting for the domain to stop.
 StopDomain.noDomainNameAllowed=No domain name allowed with --host option.
 StopDomain.nopidprev=Can not find the process ID of the server.  It is supposed to be here: {0}.  Unable to kill the process.

nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/RestartDomainCommand.java

@@ -26,6 +26,7 @@
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Supplier;
 
 import org.glassfish.api.Param;
 import org.glassfish.api.admin.CommandException;
@@ -92,7 +93,7 @@ protected void doCommand() throws CommandException {
 
         // oldPid is received from the running server.
         final Long oldPid = getServerPid();
-        final HostAndPort oldAdminAddress = getReachableAdminAddress(getDomainXml());
+        final HostAndPort oldAdminAddress = getUserProvidedAdminAddress();
         final RemoteCLICommand cmd = new RemoteCLICommand("restart-domain", programOpts, env);
         if (debug == null) {
             cmd.executeAndReturnOutput("restart-domain");
@@ -101,17 +102,17 @@ protected void doCommand() throws CommandException {
         }
 
         final Duration timeout = getRestartTimeout();
-        final Duration startTimeout = step("Waiting until domain stops.", timeout,
-            () -> waitForStop(oldPid, oldAdminAddress, timeout));
+        final Duration startTimeout = step(null, timeout,
+            () -> waitForStop(isLocal() ? oldPid : null, oldAdminAddress, timeout));
 
         final List<HostAndPort> userEndpoints = parseCustomEndpoints(customEndpoints);
         final ServerLifeSignCheck lifeSignCheck = new ServerLifeSignCheck("domain " + getDomainName(),
             printServerOutput, checkPidFile, checkProcessAlive, checkAdminEndpoint, userEndpoints);
-        final List<HostAndPort> newAdminEndpoints = getAdminAddress(getDomainXml(), "server");
-        final List<HostAndPort> adminEndpoints = newAdminEndpoints.isEmpty()
-            ? List.of(getUserProvidedAdminAddress())
-            : newAdminEndpoints;
-        final String report = waitForStart(oldPid, lifeSignCheck, adminEndpoints, startTimeout);
+        final Supplier<List<HostAndPort>> adminEndpointsSupplier = () -> {
+            final List<HostAndPort> newAdminEndpoints = getAdminAddresses(getDomainXml(), "server");
+            return newAdminEndpoints.isEmpty() ? List.of(getUserProvidedAdminAddress()) : newAdminEndpoints;
+        };
+        final String report = waitForStart(oldPid, lifeSignCheck, adminEndpointsSupplier, startTimeout);
         logger.info(report);
     }