From 3bb395f2c2b59d5eafcae2c54c8b250c11b8924d Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 30 Nov 2023 11:29:23 +0000 Subject: [PATCH 01/27] Simpler version of data in --- .../messages/scp/SendMCDataRequest.java | 70 +++ .../spinnaker/messages/sdp/SDPPort.java | 4 +- .../spinnaker/transceiver/Transceiver.java | 10 + .../transceiver/TransceiverInterface.java | 28 + .../WriteMemoryByMulticastProcess.java | 142 +++++ .../front_end/CommandLineInterface.java | 74 ++- .../dse/FastMCExecuteDataSpecification.java | 493 ++++++++++++++++++ 7 files changed, 819 insertions(+), 2 deletions(-) create mode 100644 SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java create mode 100644 SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java create mode 100644 SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java new file mode 100644 index 0000000000..1d6f7d299b --- /dev/null +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 The University of Manchester + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.manchester.spinnaker.messages.scp; + +import static uk.ac.manchester.spinnaker.messages.sdp.SDPHeader.Flag.REPLY_EXPECTED; +import static uk.ac.manchester.spinnaker.messages.sdp.SDPPort.COPY_DATA_IN_PORT; +import static uk.ac.manchester.spinnaker.messages.scp.SCPCommand.CMD_WRITE; +import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; + + +import java.nio.ByteBuffer; + +import uk.ac.manchester.spinnaker.machine.HasCoreLocation; +import uk.ac.manchester.spinnaker.machine.MemoryLocation; +import uk.ac.manchester.spinnaker.messages.model.UnexpectedResponseCodeException; +import uk.ac.manchester.spinnaker.messages.sdp.SDPHeader; +import uk.ac.manchester.spinnaker.messages.sdp.SDPLocation; + +/** + * A command message to the Data In port to write multicast data. + */ +public class SendMCDataRequest extends SCPRequest { + /** + * @param core + * Where to send the request. + * @param targetCore + * The target core of the write. + */ + public SendMCDataRequest(HasCoreLocation core, HasCoreLocation targetCore, + MemoryLocation baseAddress, ByteBuffer data) { + super(header(core), CMD_WRITE, baseAddress.address, + targetCore.getX() << 16 | targetCore.getY(), + data.remaining() / WORD_SIZE, data); + } + + + /** + * Make a variant of SDP header that talks to the packet reinjector. It + * always wants a reply and always talks to a particular SDP port + * (the port for the reinjector). + * + * @param core + * The SpiNNaker core that we want to talk to. Should be running + * the extra monitor core (not checked). + * @return The SDP header. + */ + private static SDPHeader header(HasCoreLocation core) { + return new SDPHeader(REPLY_EXPECTED, new SDPLocation(core), + COPY_DATA_IN_PORT); + } + + @Override + public EmptyResponse getSCPResponse(ByteBuffer buffer) + throws UnexpectedResponseCodeException { + return new EmptyResponse("Copy Data In", CMD_WRITE, buffer); + } +} diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/sdp/SDPPort.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/sdp/SDPPort.java index fb8bb52016..aabfd12a4c 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/sdp/SDPPort.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/sdp/SDPPort.java @@ -30,7 +30,9 @@ public enum SDPPort { /** Extra monitor core data transfer functionality. */ EXTRA_MONITOR_CORE_DATA_SPEED_UP(5), /** Messages directed at the packet gatherer for the speed up protocols. */ - GATHERER_DATA_SPEED_UP(6); + GATHERER_DATA_SPEED_UP(6), + /** Messages directed at the data in loader to simply load data. */ + COPY_DATA_IN_PORT(7); /** The port ID. */ public final int value; diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java index eee1d9b7c1..60a71d9077 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java @@ -2086,6 +2086,16 @@ public void writeMemoryFlood(MemoryLocation baseAddress, ByteBuffer data) } } + @Override + @ParallelSafe + public void writeMemoryMulticast(HasCoreLocation core, + HasCoreLocation targetCore, MemoryLocation baseAddress, + ByteBuffer data) + throws IOException, ProcessException, InterruptedException { + new WriteMemoryByMulticastProcess(scpSelector, this).writeMemory( + core, targetCore, baseAddress, data); + }; + @Override @CheckReturnValue @ParallelSafe diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java index 1aceb5d99d..232587757a 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java @@ -1866,6 +1866,34 @@ void writeMemory(@Valid HasCoreLocation core, @NotNull MemoryLocation baseAddress, @NotNull ByteBuffer data) throws IOException, ProcessException, InterruptedException; + /** + * Write to the board via multicast on the Ethernet chip. + * + * @param core + * The coordinates of the Ethernet core containing the advanced + * monitor support + * @param targetCore + * The coordinates of the core where the memory is that is to be + * written to + * @param baseAddress + * The address in SDRAM where the region of memory is to be + * written + * @param data + * The data that is to be written. The data should be from the + * position to the limit. + * @throws IOException + * If anything goes wrong with networking. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If the communications were interrupted. + */ + @ParallelSafe + void writeMemoryMulticast(@Valid HasCoreLocation core, + @Valid HasCoreLocation targetCore, + @NotNull MemoryLocation baseAddress, @NotNull ByteBuffer data) + throws IOException, ProcessException, InterruptedException; + /** * Write to the user0 register of a core. * diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java new file mode 100644 index 0000000000..55ba81f523 --- /dev/null +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2018 The University of Manchester + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.manchester.spinnaker.transceiver; + +import static java.lang.Math.max; +import static java.nio.ByteBuffer.allocate; +import static uk.ac.manchester.spinnaker.messages.Constants.UDP_MESSAGE_MAX_SIZE; +import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.read; +import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +import uk.ac.manchester.spinnaker.connections.ConnectionSelector; +import uk.ac.manchester.spinnaker.connections.SCPConnection; +import uk.ac.manchester.spinnaker.machine.HasCoreLocation; +import uk.ac.manchester.spinnaker.machine.MemoryLocation; +import uk.ac.manchester.spinnaker.messages.scp.SendMCDataRequest; + +/** + * Write to memory on SpiNNaker via multicast (data in only) + */ +class WriteMemoryByMulticastProcess extends TxrxProcess { + /** + * @param connectionSelector + * How to select how to communicate. + * @param retryTracker + * Object used to track how many retries were used in an + * operation. May be {@code null} if no suck tracking is + * required. + */ + WriteMemoryByMulticastProcess( + ConnectionSelector connectionSelector, + RetryTracker retryTracker) { + super(connectionSelector, retryTracker); + } + + /** + * @param connectionSelector + * How to select how to communicate. + * @param numChannels + * The number of parallel communications to support + * @param retryTracker + * Object used to track how many retries were used in an + * operation. May be {@code null} if no suck tracking is + * required. + */ + WriteMemoryByMulticastProcess( + ConnectionSelector connectionSelector, + int numChannels, RetryTracker retryTracker) { + super(connectionSelector, SCP_RETRIES, SCP_TIMEOUT, numChannels, + max(numChannels / 2, 1), retryTracker); + } + + /** + * Write to memory. + * + * @param core + * The location to send the message to. + * @param targetCore + * The target to write the data to. + * @param baseAddress + * The base address to write. + * @param data + * The overall block of memory to write + * @param msgProvider + * The way to create messages to send to do the writing. + * @throws IOException + * If anything goes wrong with networking. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If the communications were interrupted. + */ + public void writeMemory( + HasCoreLocation core, HasCoreLocation targetCore, + MemoryLocation baseAddress, ByteBuffer data) + throws IOException, ProcessException, InterruptedException { + var writePosition = baseAddress; + for (var bb : sliceUp(data, UDP_MESSAGE_MAX_SIZE)) { + sendRequest(new SendMCDataRequest(core, targetCore, writePosition, + bb)); + writePosition = writePosition.add(bb.remaining()); + } + finishBatch(); + } + + /** + * Write to memory. + * + * @param core + * The location to send the message to. + * @param targetCore + * The target to write the data to. + * @param baseAddress + * The base address to write. + * @param data + * The stream of data to write. + * @param bytesToWrite + * The number of bytes to read from the stream and transfer. + * @param msgProvider + * The way to create messages to send to do the writing. + * @throws IOException + * If anything goes wrong with networking or the input stream. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If the communications were interrupted. + */ + public void writeMemory( + HasCoreLocation core, HasCoreLocation targetCore, + MemoryLocation baseAddress, InputStream data) + throws IOException, ProcessException, InterruptedException { + var writePosition = baseAddress; + while (true) { + // One buffer per message; lifetime extends until batch end + var tmp = read(data, allocate(UDP_MESSAGE_MAX_SIZE), + UDP_MESSAGE_MAX_SIZE); + if (tmp == null) { + break; + } + sendRequest(new SendMCDataRequest(core, targetCore, writePosition, + tmp)); + writePosition = writePosition.add(tmp.remaining()); + } + finishBatch(); + } +} diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/CommandLineInterface.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/CommandLineInterface.java index 01f5602380..a92e7c1612 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/CommandLineInterface.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/CommandLineInterface.java @@ -23,8 +23,8 @@ import static uk.ac.manchester.spinnaker.alloc.client.SpallocClientFactory.getJobFromProxyInfo; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DOWNLOAD_DESC; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DSE_APP_DESC; -import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DSE_DESC; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DSE_MON_DESC; +import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DSE_MON_DESC_MC; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DSE_SYS_DESC; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.GATHER_DESC; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.IOBUF_DESC; @@ -72,6 +72,7 @@ import uk.ac.manchester.spinnaker.front_end.download.request.Gather; import uk.ac.manchester.spinnaker.front_end.download.request.Placement; import uk.ac.manchester.spinnaker.front_end.dse.FastExecuteDataSpecification; +import uk.ac.manchester.spinnaker.front_end.dse.FastMCExecuteDataSpecification; import uk.ac.manchester.spinnaker.front_end.dse.HostExecuteDataSpecification; import uk.ac.manchester.spinnaker.front_end.iobuf.IobufRequest; import uk.ac.manchester.spinnaker.front_end.iobuf.IobufRetriever; @@ -209,6 +210,21 @@ FastExecuteDataSpecification create(TransceiverInterface txrx, */ static FastDSEFactory fastFactory = FastExecuteDataSpecification::new; + @FunctionalInterface + interface FastMCDSEFactory { + FastMCExecuteDataSpecification create(TransceiverInterface txrx, + Machine machine, List gatherers, File reportDir, + DSEDatabaseEngine db) + throws IOException, SpinnmanException, StorageException, + ExecutionException, InterruptedException, URISyntaxException; + } + + /** + * Makes {@link FastMCExecuteDataSpecification} instances. Allows for + * injection of debugging tooling. + */ + static FastMCDSEFactory fastMCFactory = FastMCExecuteDataSpecification::new; + /** * Run the data specifications in parallel. * @@ -304,6 +320,54 @@ public void runDSEForAppCoresUploadingViaMonitorStreaming( } } + /** + * Run the data specifications in parallel using monitors and multicast. + * + * @param gatherers + * List of descriptions of gatherers. + * @param machine + * Description of overall machine. + * @param dsFile + * Path to the dataspec database + * @param runFolder + * Directory containing per-run information. + * @param reportFolder + * Directory containing reports. If {@link Optional#empty()}, no + * report will be written. + * @throws IOException + * If the communications fail. + * @throws SpinnmanException + * If a BMP is uncontactable or SpiNNaker rejects a message. + * @throws StorageException + * If the database is in an illegal state. + * @throws ExecutionException + * If there was a problem in the parallel queue. + * @throws InterruptedException + * If the wait for everything to complete is interrupted. + * @throws URISyntaxException + * If a proxy URI is provided but invalid. + */ + @Command(name = "dse_app_mon_mc", description = DSE_MON_DESC_MC) + public void runDSEForAppCoresUploadingViaMulticast( + @Mixin GatherersParam gatherers, + @Mixin MachineParam machine, + @Mixin DsFileParam dsFile, + @Mixin RunFolderParam runFolder, + @Parameters(description = REPORT, arity = "0..1", index = "3") + Optional reportFolder) + throws IOException, SpinnmanException, StorageException, + ExecutionException, InterruptedException, URISyntaxException { + setLoggerDir(runFolder.get()); + var db = getDataSpecDB(dsFile.get()); + var job = getJob(db); + + try (var txrx = getTransceiver(machine.get(), job); + var dseExec = fastMCFactory.create(txrx, machine.get(), + gatherers.get(), reportFolder.orElse(null), db)) { + dseExec.loadCores(); + } + } + /** * Retrieve IOBUFs in parallel. * @@ -770,6 +834,14 @@ interface CommandDescriptions { + "Requires system cores to be fully configured, so " + "can't be used to set up system cores."; + /** Description of {@code dse_app_mon} command. */ + String DSE_MON_DESC_MC = + "Evaluate data specifications for application cores " + + "and upload the results to SpiNNaker using the fast data " + + "streaming protocol directly with multicast. " + + "Requires system cores to be fully configured, so " + + "can't be used to set up system cores."; + /** Description of {@code dse_sys} command. */ String DSE_SYS_DESC = "Evaluate data specifications for system cores and " + "upload the results to SpiNNaker (always uses the classic " diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java new file mode 100644 index 0000000000..4a1466274a --- /dev/null +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -0,0 +1,493 @@ +/* + * Copyright (c) 2019 The University of Manchester + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.manchester.spinnaker.front_end.dse; + +import static difflib.DiffUtils.diff; +import static java.lang.Integer.toUnsignedLong; +import static java.lang.String.format; +import static java.lang.System.getProperty; +import static java.lang.System.nanoTime; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.toList; +import static org.slf4j.LoggerFactory.getLogger; +import static uk.ac.manchester.spinnaker.messages.Constants.NBBY; +import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; +import static uk.ac.manchester.spinnaker.utils.UnitConstants.MEGABYTE; +import static uk.ac.manchester.spinnaker.utils.UnitConstants.NSEC_PER_SEC; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; + +import com.google.errorprone.annotations.MustBeClosed; + +import difflib.ChangeDelta; +import difflib.Chunk; +import difflib.DeleteDelta; +import difflib.InsertDelta; +import uk.ac.manchester.spinnaker.front_end.NoDropPacketContext; +import uk.ac.manchester.spinnaker.front_end.download.request.Gather; +import uk.ac.manchester.spinnaker.front_end.download.request.Monitor; +import uk.ac.manchester.spinnaker.machine.ChipLocation; +import uk.ac.manchester.spinnaker.machine.CoreLocation; +import uk.ac.manchester.spinnaker.machine.CoreSubsets; +import uk.ac.manchester.spinnaker.machine.HasChipLocation; +import uk.ac.manchester.spinnaker.machine.HasCoreLocation; +import uk.ac.manchester.spinnaker.machine.Machine; +import uk.ac.manchester.spinnaker.machine.MemoryLocation; +import uk.ac.manchester.spinnaker.storage.DSEDatabaseEngine; +import uk.ac.manchester.spinnaker.storage.DSEStorage; +import uk.ac.manchester.spinnaker.storage.DSEStorage.Ethernet; +import uk.ac.manchester.spinnaker.storage.StorageException; +import uk.ac.manchester.spinnaker.transceiver.ProcessException; +import uk.ac.manchester.spinnaker.transceiver.TransceiverInterface; +import uk.ac.manchester.spinnaker.utils.MathUtils; + +/** + * Implementation of the Data Specification Executor that uses the Fast Data In + * protocol to upload the results to a SpiNNaker machine. + * + * @author Donal Fellows + * @author Alan Stokes + */ +public class FastMCExecuteDataSpecification extends ExecuteDataSpecification { + private static final Logger log = + getLogger(FastMCExecuteDataSpecification.class); + + private static final String SPINNAKER_COMPARE_UPLOAD = + getProperty("spinnaker.compare.upload"); + + private static final String IN_REPORT_NAME = + "speeds_gained_in_speed_up_process.tsv"; + + private final Map gathererForChip; + + private final Map monitorForChip; + + private final Map monitorsForBoard; + + private boolean writeReports = false; + + private File reportPath = null; + + /** + * Create an instance of this class. + * + * @param txrx + * The transceiver for talking to the SpiNNaker machine. + * @param machine + * The SpiNNaker machine description. + * @param gatherers + * The description of where the gatherers and monitors are. + * @param reportDir + * Where to write reports, or {@code null} if no reports are to + * be written. + * @param db + * The DSE Database. + * @throws IOException + * If IO goes wrong. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If communications are interrupted. + * @throws URISyntaxException + * If the proxy URI is provided but not valid. + * @throws StorageException + * If there is an error reading the database. + * @throws IllegalStateException + * If something really strange occurs with talking to the BMP; + * this constructor should not be doing that! + */ + @MustBeClosed + public FastMCExecuteDataSpecification(TransceiverInterface txrx, + Machine machine, List gatherers, File reportDir, + DSEDatabaseEngine db) throws IOException, ProcessException, + InterruptedException, StorageException, URISyntaxException { + super(txrx, machine, db); + if (SPINNAKER_COMPARE_UPLOAD != null) { + log.warn( + "detailed comparison of uploaded data enabled; " + + "this may destabilize the protocol"); + } + + if (reportDir != null) { + writeReports = true; + reportPath = new File(reportDir, IN_REPORT_NAME); + } + + gathererForChip = new HashMap<>(); + monitorForChip = new HashMap<>(); + monitorsForBoard = new HashMap<>(); + + buildMaps(gatherers); + } + + /** + * Construct the internal mappings for gatherers and monitors. + * + * @param gatherers + * The descriptions of whether the gatherers are located. + * @throws IOException + * If IDs can't be read from the machine for network reasons. + * @throws ProcessException + * If IDs can't be read from the machine for machine reasons. + * @throws InterruptedException + * If we are interrupted. + */ + protected void buildMaps(List gatherers) + throws IOException, ProcessException, InterruptedException { + for (var g : gatherers) { + g.updateTransactionIdFromMachine(txrx); + var gathererChip = g.asChipLocation(); + gathererForChip.put(gathererChip, g); + var boardMonitorCores = monitorsForBoard + .computeIfAbsent(gathererChip, __ -> new CoreSubsets()); + for (var m : g.getMonitors()) { + var monitorChip = m.asChipLocation(); + gathererForChip.put(monitorChip, g); + monitorForChip.put(monitorChip, m); + boardMonitorCores.addCore(m.asCoreLocation()); + } + } + } + + /** + * Execute all application data specifications that a particular connection + * knows about, storing back in the database the information collected about + * those executions. Data is transferred using the Fast Data In protocol. + *

+ * Cannot load data for system cores; those are used by the implementation + * of this protocol. + * + * @throws StorageException + * If the database can't be talked to. + * @throws IOException + * If the transceiver can't talk to its sockets. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If communications are interrupted. + * @throws IllegalStateException + * If an unexpected exception occurs in any of the parallel + * tasks. + */ + public void loadCores() + throws StorageException, IOException, ProcessException, + InterruptedException { + var storage = db.getStorageInterface(); + processTasksInParallel(storage.listEthernetsToLoad(), board -> { + return () -> loadBoard(board, storage); + }); + } + + private void loadBoard(Ethernet board, DSEStorage storage) + throws IOException, ProcessException, StorageException, + InterruptedException { + var cores = storage.listCoresToLoad(board, false); + if (cores.isEmpty()) { + log.info("no cores need loading on board; skipping"); + return; + } + log.info("loading data onto {} cores on board", cores.size()); + var gather = gathererForChip.get(board.location); + var worker = new FastBoardWorker(txrx, board, storage, gather); + for (var xyp : cores) { + worker.mallocCore(xyp); + } + try (var routers = worker.systemRouterTables(); + var context = worker.dontDropPackets(gather)) { + for (var xyp : cores) { + worker.loadCore(xyp); + } + log.info("finished sending data in for this board"); + } catch (Exception e) { + log.warn("failure in core loading", e); + throw e; + } + } + + /** + * Opens a file for writing text. + * + * @param file + * The file to open + * @param append + * Whether to open in append mode; if {@code false}, the file + * will be created or overwritten. + * @return The stream to use to do the writing. + * @throws IOException + * If anything goes wrong with creating or opening the file. + */ + private static PrintWriter open(File file, boolean append) + throws IOException { + return new PrintWriter(new BufferedWriter(new OutputStreamWriter( + new FileOutputStream(file, append), UTF_8))); + } + + /** + * Writes (part of) the report describing what data transfer rates were + * achieved. + * + * @param chip + * Which chip was the data bound for? + * @param timeDiff + * How long did the transfer take, in nanoseconds. + * @param size + * How many bytes were transferred? + * @param baseAddress + * Where were the bytes written to? + * @param missingNumbers + * What were the missing sequence numbers at each stage. + * @throws IOException + * If IO fails. + */ + public synchronized void writeReport(HasChipLocation chip, long timeDiff, + int size, MemoryLocation baseAddress, Object missingNumbers) + throws IOException { + if (!reportPath.exists()) { + try (var w = open(reportPath, false)) { + w.println("x" + "\ty" + "\tSDRAM address" + "\tsize/bytes" + + "\ttime taken/s" + "\ttransfer rate/(Mb/s)" + + "\tmissing sequence numbers"); + } + } + + float timeTaken = timeDiff / (float) NSEC_PER_SEC; + float megabits = (size * (long) NBBY) / (float) MEGABYTE; + String mbs; + if (timeDiff == 0) { + mbs = "unknown, below threshold"; + } else { + mbs = format("%f", megabits / timeTaken); + } + try (var w = open(reportPath, true)) { + w.printf("%d\t%d\t%s\t%d\t%f\t%s\t%s\n", chip.getX(), chip.getY(), + baseAddress, toUnsignedLong(size), timeTaken, mbs, + missingNumbers); + } + } + + private static void compareBuffers(ByteBuffer original, + ByteBuffer downloaded) { + for (int i = 0; i < original.remaining(); i++) { + if (original.get(i) != downloaded.get(i)) { + log.error("downloaded buffer contents different"); + for (var delta : diff(list(original), list(downloaded)) + .getDeltas()) { + if (delta instanceof ChangeDelta) { + var delete = delta.getOriginal(); + var insert = delta.getRevised(); + log.warn( + "swapped {} bytes (SCP) for {} (gather) " + + "at {}->{}", + delete.getLines().size(), + insert.getLines().size(), delete.getPosition(), + insert.getPosition()); + log.info("change {} -> {}", describeChunk(delete), + describeChunk(insert)); + } else if (delta instanceof DeleteDelta) { + var delete = delta.getOriginal(); + log.warn("gather deleted {} bytes at {}", + delete.getLines().size(), delete.getPosition()); + log.info("delete {}", describeChunk(delete)); + } else if (delta instanceof InsertDelta) { + var insert = delta.getRevised(); + log.warn("gather inserted {} bytes at {}", + insert.getLines().size(), insert.getPosition()); + log.info("insert {}", describeChunk(insert)); + } + } + break; + } + } + } + + private static List list(ByteBuffer buffer) { + return sliceUp(buffer, 1).map(ByteBuffer::get).toList(); + } + + private static List describeChunk(Chunk chunk) { + return chunk.getLines().stream().map(MathUtils::hexbyte) + .collect(toList()); + } + + /** + * The worker class that handles a particular board of a SpiNNaker machine. + * Instances of this class are only ever used from a single thread. + * + * @author Donal Fellows + * @author Alan Stokes + */ + private class FastBoardWorker extends BoardWorker { + private HasCoreLocation ethernet; + + + @MustBeClosed + @SuppressWarnings("MustBeClosed") + FastBoardWorker(TransceiverInterface txrx, Ethernet board, + DSEStorage storage, Gather gather) + throws IOException, ProcessException, InterruptedException, + StorageException { + super(txrx, board, storage); + this.ethernet = new CoreLocation(board.location, gather.getP()); + } + + /** + * A list of bitfields. Knows how to install and uninstall itself from + * the general execution flow. + * + * @author Donal Fellows + */ + private class MissingRecorder { + + + /** + * Issue the report based on what we recorded, if appropriate. + * + * @param core + * What core were we recording for? + * @param time + * How long did the loading take? + * @param size + * How much data was moved? + * @param addr + * Where on the core was the data moved to? + * @throws IOException + * If anything goes wrong with writing. + */ + void report(HasCoreLocation core, long time, int size, + MemoryLocation addr) throws IOException { + if (writeReports) { + writeReport(core, time, size, addr, this); + } + } + } + + /** + * Writes the contents of a region. Caller is responsible for ensuring + * this method has work to do. + * + * @param core + * Which core to write to. Does not need to refer to a + * monitor core. + * @param region + * The region to write. + * @param baseAddress + * Where to write the region. + * @param gather + * The information about where messages are routed via. + * @return How many bytes were actually written. + * @throws IOException + * If anything goes wrong with I/O. + * @throws ProcessException + * If SCAMP rejects the request. + * @throws InterruptedException + * If communications are interrupted. + */ + @Override + protected int writeRegion(HasCoreLocation core, ByteBuffer content, + MemoryLocation baseAddress) + throws IOException, ProcessException, InterruptedException { + int written = content.remaining(); + var recorder = new MissingRecorder(); + long start = nanoTime(); + fastWrite(core, baseAddress, content); + long end = nanoTime(); + recorder.report(core, end - start, content.limit(), baseAddress); + if (SPINNAKER_COMPARE_UPLOAD != null) { + var readBack = txrx.readMemory( + core, baseAddress, content.remaining()); + compareBuffers(content, readBack); + } + return written; + } + + /** + * Put the board in don't-drop-packets mode. + * + * @param core + * The core location of the gatherer for the board to set to + * don't drop packets. + * @return An object that, when closed, will put the board back in + * standard mode. + * @throws IOException + * If anything goes wrong with communication. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If communications are interrupted. + */ + @MustBeClosed + NoDropPacketContext dontDropPackets(Gather core) + throws IOException, ProcessException, InterruptedException { + return new NoDropPacketContext(txrx, + monitorsForBoard.get(board.location), core); + } + + /** + * Install the system router tables across the board. + * + * @return An object that, when closed, will put the board back in + * application mode. + * @throws IOException + * If anything goes wrong with communication. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If communications are interrupted. + */ + @MustBeClosed + SystemRouterTableContext systemRouterTables() + throws IOException, ProcessException, InterruptedException { + return new SystemRouterTableContext(txrx, + monitorsForBoard.get(board.location)); + } + + /** + * This is the implementation of the actual fast data in protocol. + * + * @param core + * Where the data is going to. + * @param baseAddress + * Whether the data will be written. + * @param data + * The data to be written. + * @throws IOException + * If IO fails. + * @throws InterruptedException + * If communications are interrupted. + */ + private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, + ByteBuffer data) + throws IOException, InterruptedException { + try { + txrx.writeMemoryMulticast(ethernet, core, baseAddress, data); + } catch (ProcessException e) { + throw new IOException(e); + } + } + } +} From ca54bbd297733335e3215ab1209dcae3f00bb3b4 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 30 Nov 2023 11:47:06 +0000 Subject: [PATCH 02/27] Style --- .../spinnaker/messages/scp/SendMCDataRequest.java | 9 ++++++++- .../ac/manchester/spinnaker/transceiver/Transceiver.java | 2 +- .../transceiver/WriteMemoryByMulticastProcess.java | 8 +------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java index 1d6f7d299b..64dbf38bd7 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java @@ -33,16 +33,23 @@ * A command message to the Data In port to write multicast data. */ public class SendMCDataRequest extends SCPRequest { + /** Shift of x in the coordinates for arg2. */ + private static final int X_SHIFT = 16; + /** * @param core * Where to send the request. * @param targetCore * The target core of the write. + * @param baseAddress + * The address to write to on the target core. + * @param data + * The data to write. */ public SendMCDataRequest(HasCoreLocation core, HasCoreLocation targetCore, MemoryLocation baseAddress, ByteBuffer data) { super(header(core), CMD_WRITE, baseAddress.address, - targetCore.getX() << 16 | targetCore.getY(), + (targetCore.getX() << X_SHIFT) | targetCore.getY(), data.remaining() / WORD_SIZE, data); } diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java index 60a71d9077..edfed1d6eb 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java @@ -2094,7 +2094,7 @@ public void writeMemoryMulticast(HasCoreLocation core, throws IOException, ProcessException, InterruptedException { new WriteMemoryByMulticastProcess(scpSelector, this).writeMemory( core, targetCore, baseAddress, data); - }; + } @Override @CheckReturnValue diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index 55ba81f523..378264d601 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -32,7 +32,7 @@ import uk.ac.manchester.spinnaker.messages.scp.SendMCDataRequest; /** - * Write to memory on SpiNNaker via multicast (data in only) + * Write to memory on SpiNNaker via multicast (data in only). */ class WriteMemoryByMulticastProcess extends TxrxProcess { /** @@ -77,8 +77,6 @@ class WriteMemoryByMulticastProcess extends TxrxProcess { * The base address to write. * @param data * The overall block of memory to write - * @param msgProvider - * The way to create messages to send to do the writing. * @throws IOException * If anything goes wrong with networking. * @throws ProcessException @@ -110,10 +108,6 @@ public void writeMemory( * The base address to write. * @param data * The stream of data to write. - * @param bytesToWrite - * The number of bytes to read from the stream and transfer. - * @param msgProvider - * The way to create messages to send to do the writing. * @throws IOException * If anything goes wrong with networking or the input stream. * @throws ProcessException From af3f58d2e61f6c4f6254f6a6d376b3b58c77cf82 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 30 Nov 2023 11:50:31 +0000 Subject: [PATCH 03/27] Bigger timeout for packets --- .../spinnaker/transceiver/WriteMemoryByMulticastProcess.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index 378264d601..f8967152ee 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -35,6 +35,9 @@ * Write to memory on SpiNNaker via multicast (data in only). */ class WriteMemoryByMulticastProcess extends TxrxProcess { + /** Timeout for a write request; longer as the write can take some time. */ + private static final int TIMEOUT = 10000; + /** * @param connectionSelector * How to select how to communicate. @@ -62,7 +65,7 @@ class WriteMemoryByMulticastProcess extends TxrxProcess { WriteMemoryByMulticastProcess( ConnectionSelector connectionSelector, int numChannels, RetryTracker retryTracker) { - super(connectionSelector, SCP_RETRIES, SCP_TIMEOUT, numChannels, + super(connectionSelector, SCP_RETRIES, TIMEOUT, numChannels, max(numChannels / 2, 1), retryTracker); } From 08db6574ca8a425f467a6e19b1de3b13f045847d Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 30 Nov 2023 12:44:42 +0000 Subject: [PATCH 04/27] Style --- .../ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java index 64dbf38bd7..8279601023 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java @@ -53,7 +53,6 @@ public SendMCDataRequest(HasCoreLocation core, HasCoreLocation targetCore, data.remaining() / WORD_SIZE, data); } - /** * Make a variant of SDP header that talks to the packet reinjector. It * always wants a reply and always talks to a particular SDP port From a8b299f2196912b0a61e470b4947e6b6def99e64 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 30 Nov 2023 14:15:47 +0000 Subject: [PATCH 05/27] Try just one at at time --- .../transceiver/WriteMemoryByMulticastProcess.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index f8967152ee..5007458685 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -35,9 +35,13 @@ * Write to memory on SpiNNaker via multicast (data in only). */ class WriteMemoryByMulticastProcess extends TxrxProcess { + /** Timeout for a write request; longer as the write can take some time. */ private static final int TIMEOUT = 10000; + /** The number of simultaneous messages that can be in progress. */ + private static final int N_CHANNELS = 1; + /** * @param connectionSelector * How to select how to communicate. @@ -49,7 +53,8 @@ class WriteMemoryByMulticastProcess extends TxrxProcess { WriteMemoryByMulticastProcess( ConnectionSelector connectionSelector, RetryTracker retryTracker) { - super(connectionSelector, retryTracker); + super(connectionSelector, SCP_RETRIES, TIMEOUT, N_CHANNELS, N_CHANNELS, + retryTracker); } /** From 33720a220700f2b065ec78f239d499298f88cc36 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 30 Nov 2023 14:27:28 +0000 Subject: [PATCH 06/27] Style --- .../transceiver/WriteMemoryByMulticastProcess.java | 6 +++--- .../front_end/dse/FastMCExecuteDataSpecification.java | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index 5007458685..c805627364 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -40,7 +40,7 @@ class WriteMemoryByMulticastProcess extends TxrxProcess { private static final int TIMEOUT = 10000; /** The number of simultaneous messages that can be in progress. */ - private static final int N_CHANNELS = 1; + private static final int N_CHANNELS = 8; /** * @param connectionSelector @@ -53,8 +53,8 @@ class WriteMemoryByMulticastProcess extends TxrxProcess { WriteMemoryByMulticastProcess( ConnectionSelector connectionSelector, RetryTracker retryTracker) { - super(connectionSelector, SCP_RETRIES, TIMEOUT, N_CHANNELS, N_CHANNELS, - retryTracker); + super(connectionSelector, SCP_RETRIES, TIMEOUT, N_CHANNELS, + N_CHANNELS - 1, retryTracker); } /** diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 4a1466274a..f7951d3ee3 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -344,7 +344,6 @@ private static List describeChunk(Chunk chunk) { private class FastBoardWorker extends BoardWorker { private HasCoreLocation ethernet; - @MustBeClosed @SuppressWarnings("MustBeClosed") FastBoardWorker(TransceiverInterface txrx, Ethernet board, @@ -361,7 +360,7 @@ private class FastBoardWorker extends BoardWorker { * * @author Donal Fellows */ - private class MissingRecorder { + private final class MissingRecorder { /** From a146979557d53a5a55be56bbbb39080286d4178c Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 30 Nov 2023 14:36:29 +0000 Subject: [PATCH 07/27] No need to close --- .../spinnaker/front_end/dse/FastMCExecuteDataSpecification.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index f7951d3ee3..f8d1b546e2 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -344,8 +344,6 @@ private static List describeChunk(Chunk chunk) { private class FastBoardWorker extends BoardWorker { private HasCoreLocation ethernet; - @MustBeClosed - @SuppressWarnings("MustBeClosed") FastBoardWorker(TransceiverInterface txrx, Ethernet board, DSEStorage storage, Gather gather) throws IOException, ProcessException, InterruptedException, From c225f47a18f6d38f08f41a1b70d98b8c0a42908d Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 30 Nov 2023 14:42:02 +0000 Subject: [PATCH 08/27] Back to 1 for testing --- .../spinnaker/transceiver/WriteMemoryByMulticastProcess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index c805627364..d3397e1c72 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -40,7 +40,7 @@ class WriteMemoryByMulticastProcess extends TxrxProcess { private static final int TIMEOUT = 10000; /** The number of simultaneous messages that can be in progress. */ - private static final int N_CHANNELS = 8; + private static final int N_CHANNELS = 1; /** * @param connectionSelector From 22d4a2248aa922c20c6e6ecb1534072dfe4f2c65 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Fri, 1 Dec 2023 08:36:31 +0000 Subject: [PATCH 09/27] Use board local coordinates --- .../dse/FastMCExecuteDataSpecification.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index f8d1b546e2..4ac6d10a7f 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -481,7 +481,17 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, ByteBuffer data) throws IOException, InterruptedException { try { - txrx.writeMemoryMulticast(ethernet, core, baseAddress, data); + int boardLocalX = core.getX() - ethernet.getX(); + if (boardLocalX < 0) { + boardLocalX += machine.maxChipX() + 1; + } + int boardLocalY = core.getY() - ethernet.getY(); + if (boardLocalY < 0) { + boardLocalY += machine.maxChipY() + 1; + } + var boardLocal = new CoreLocation(boardLocalX, boardLocalY, + core.getP()); + txrx.writeMemoryMulticast(ethernet, boardLocal, baseAddress, data); } catch (ProcessException e) { throw new IOException(e); } From a2e8627a2160acc4bafedec85e8ae6cc2f04392c Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Fri, 1 Dec 2023 08:46:39 +0000 Subject: [PATCH 10/27] Style --- .../front_end/dse/FastMCExecuteDataSpecification.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 4ac6d10a7f..f64b6bb073 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -491,7 +491,8 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, } var boardLocal = new CoreLocation(boardLocalX, boardLocalY, core.getP()); - txrx.writeMemoryMulticast(ethernet, boardLocal, baseAddress, data); + txrx.writeMemoryMulticast(ethernet, boardLocal, baseAddress, + data); } catch (ProcessException e) { throw new IOException(e); } From 7a19bef60d974f95eef52d6b784cfaa57468bdc9 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Fri, 1 Dec 2023 09:21:02 +0000 Subject: [PATCH 11/27] Does more channels help --- .../spinnaker/transceiver/WriteMemoryByMulticastProcess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index d3397e1c72..c805627364 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -40,7 +40,7 @@ class WriteMemoryByMulticastProcess extends TxrxProcess { private static final int TIMEOUT = 10000; /** The number of simultaneous messages that can be in progress. */ - private static final int N_CHANNELS = 1; + private static final int N_CHANNELS = 8; /** * @param connectionSelector From df1614bf7a1796c8139eff6c4cbaf12ba3216fa4 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Tue, 5 Dec 2023 08:30:43 +0000 Subject: [PATCH 12/27] Move data through a stream --- .../spinnaker/transceiver/Transceiver.java | 10 +++++++ .../transceiver/TransceiverInterface.java | 27 +++++++++++++++++++ .../dse/FastMCExecuteDataSpecification.java | 13 +++++++-- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java index edfed1d6eb..e61ec6b4bd 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java @@ -2096,6 +2096,16 @@ public void writeMemoryMulticast(HasCoreLocation core, core, targetCore, baseAddress, data); } + @Override + @ParallelSafe + public void writeMemoryMulticast(HasCoreLocation core, + HasCoreLocation targetCore, MemoryLocation baseAddress, + InputStream data) + throws IOException, ProcessException, InterruptedException { + new WriteMemoryByMulticastProcess(scpSelector, this).writeMemory( + core, targetCore, baseAddress, data); + } + @Override @CheckReturnValue @ParallelSafe diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java index 232587757a..95f5cb4f29 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java @@ -1894,6 +1894,33 @@ void writeMemoryMulticast(@Valid HasCoreLocation core, @NotNull MemoryLocation baseAddress, @NotNull ByteBuffer data) throws IOException, ProcessException, InterruptedException; + /** + * Write to the board via multicast on the Ethernet chip. + * + * @param core + * The coordinates of the Ethernet core containing the advanced + * monitor support + * @param targetCore + * The coordinates of the core where the memory is that is to be + * written to + * @param baseAddress + * The address in SDRAM where the region of memory is to be + * written + * @param data + * The stream of data to be written. + * @throws IOException + * If anything goes wrong with networking. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If the communications were interrupted. + */ + @ParallelSafe + void writeMemoryMulticast(@Valid HasCoreLocation core, + @Valid HasCoreLocation targetCore, + @NotNull MemoryLocation baseAddress, @NotNull InputStream data) + throws IOException, ProcessException, InterruptedException; + /** * Write to the user0 register of a core. * diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index f64b6bb073..8b0fae95d5 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -33,6 +33,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; import java.io.PrintWriter; import java.net.URISyntaxException; import java.nio.ByteBuffer; @@ -480,7 +482,14 @@ SystemRouterTableContext systemRouterTables() private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, ByteBuffer data) throws IOException, InterruptedException { - try { + + try (var input = new PipedInputStream(data.remaining()); + var output = new PipedOutputStream(input)) { + byte[] transfer = new byte[data.remaining()]; + data.get(transfer); + output.write(transfer); + output.flush(); + output.close(); int boardLocalX = core.getX() - ethernet.getX(); if (boardLocalX < 0) { boardLocalX += machine.maxChipX() + 1; @@ -492,7 +501,7 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, var boardLocal = new CoreLocation(boardLocalX, boardLocalY, core.getP()); txrx.writeMemoryMulticast(ethernet, boardLocal, baseAddress, - data); + input); } catch (ProcessException e) { throw new IOException(e); } From 9b43acf096eb36f3f355a316a20071bb80051f0d Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 07:51:54 +0000 Subject: [PATCH 13/27] Debugging... need thread-safe multi-channel system! --- .../messages/scp/SendMCDataRequest.java | 9 +- .../spinnaker/transceiver/Transceiver.java | 9 ++ .../transceiver/TransceiverInterface.java | 21 ++++ .../spinnaker/transceiver/TxrxProcess.java | 19 ++- .../WriteMemoryByMulticastProcess.java | 53 +++++++++ .../dse/FastMCExecuteDataSpecification.java | 110 ++++++++++++------ 6 files changed, 177 insertions(+), 44 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java index 8279601023..23431d2a17 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; +import uk.ac.manchester.spinnaker.machine.HasChipLocation; import uk.ac.manchester.spinnaker.machine.HasCoreLocation; import uk.ac.manchester.spinnaker.machine.MemoryLocation; import uk.ac.manchester.spinnaker.messages.model.UnexpectedResponseCodeException; @@ -39,17 +40,17 @@ public class SendMCDataRequest extends SCPRequest { /** * @param core * Where to send the request. - * @param targetCore - * The target core of the write. + * @param targetChip + * The target chip of the write. * @param baseAddress * The address to write to on the target core. * @param data * The data to write. */ - public SendMCDataRequest(HasCoreLocation core, HasCoreLocation targetCore, + public SendMCDataRequest(HasCoreLocation core, HasChipLocation targetChip, MemoryLocation baseAddress, ByteBuffer data) { super(header(core), CMD_WRITE, baseAddress.address, - (targetCore.getX() << X_SHIFT) | targetCore.getY(), + (targetChip.getX() << X_SHIFT) | targetChip.getY(), data.remaining() / WORD_SIZE, data); } diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java index e61ec6b4bd..7596e32feb 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java @@ -2106,6 +2106,15 @@ public void writeMemoryMulticast(HasCoreLocation core, core, targetCore, baseAddress, data); } + @Override + @ParallelSafe + public void writeMemoryMulticastStream(HasCoreLocation core, + InputStream data) + throws IOException, ProcessException, InterruptedException { + new WriteMemoryByMulticastProcess(scpSelector, this).writeMemoryStream( + core, data); + } + @Override @CheckReturnValue @ParallelSafe diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java index 95f5cb4f29..2c7769e16c 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java @@ -1921,6 +1921,27 @@ void writeMemoryMulticast(@Valid HasCoreLocation core, @NotNull MemoryLocation baseAddress, @NotNull InputStream data) throws IOException, ProcessException, InterruptedException; + /** + * Write to the board via multicast on the Ethernet chip from a stream which + * includes headers to indicate where data is to be streamed. + * + * @param core + * The coordinates of the Ethernet core containing the advanced + * monitor support + * @param data + * The stream of data to be written. + * @throws IOException + * If anything goes wrong with networking. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If the communications were interrupted. + */ + @ParallelSafe + void writeMemoryMulticastStream(@Valid HasCoreLocation core, + @NotNull InputStream data) + throws IOException, ProcessException, InterruptedException; + /** * Write to the user0 register of a core. * diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java index e36e3b5216..8fffbde69f 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java @@ -303,8 +303,16 @@ protected TxrxProcess( * @return The pipeline instance. */ private RequestPipeline pipeline(SCPRequest request) { - return requestPipelines.computeIfAbsent( - selector.getNextConnection(request), RequestPipeline::new); + System.err.println("Finding pipeline for request " + request); + var connection = selector.getNextConnection(request); + if (!requestPipelines.containsKey(connection)) { + System.err.println("Creating pipeline for connection " + connection); + var pipeline = new RequestPipeline(connection); + requestPipelines.put(connection, pipeline); + return pipeline; + } + System.err.println("Using existing pipeline for connection " + connection); + return requestPipelines.get(connection); } /** @@ -329,6 +337,7 @@ private void resetFailureState() { protected final void finishBatch() throws ProcessException, IOException, InterruptedException { for (var pipe : requestPipelines.values()) { + System.err.println("Finishing pipe " + pipe); pipe.finish(); } if (failure != null) { @@ -566,6 +575,7 @@ public void send() throws IOException { } log.debug("Sending request {} with connection {}", request, connection); + System.err.println("Sending sequence " + seq + " with " + connection); switch (request.sdpHeader.getFlags()) { case REPLY_EXPECTED: case REPLY_EXPECTED_NO_P2P: @@ -816,6 +826,7 @@ private void multiRetrieve(int numPacketsOutstanding) // While there are still more packets in progress than some // threshold while (numOutstandingRequests() > numPacketsOutstanding) { + System.err.println("Waiting for " + numOutstandingRequests() + " requests of " + numPacketsOutstanding); try { // Receive the next response singleRetrieve(); @@ -829,6 +840,7 @@ private void singleRetrieve() throws IOException, InterruptedException { // Receive the next response log.debug("{}: Connection {} waiting for message... timeout of {}", this, connection, packetTimeout); + System.err.println("Waiting for message from connection " + connection); var msg = connection.receiveSCPResponse(packetTimeout); if (log.isDebugEnabled()) { log.debug( @@ -836,10 +848,12 @@ private void singleRetrieve() throws IOException, InterruptedException { this, connection, msg.getResult(), msg.getSequenceNumber()); } + System.err.println("Connection " + connection + " received message seq " + msg.getSequenceNumber()); var req = getRequestForResult(msg); // Only process responses which have matching requests if (req == null) { + System.err.println("Connection " + connection + " Unknown sequence!"); log.info("discarding message with unknown sequence number: {}", msg.getSequenceNumber()); if (log.isDebugEnabled()) { @@ -854,6 +868,7 @@ private void singleRetrieve() throws IOException, InterruptedException { // If the response can be retried, retry it if (msg.isRetriable()) { try { + System.err.println("Connection " + connection + " resending..."); resend(req, msg.getResult(), msg.getSequenceNumber()); numRetryCodeResent++; } catch (SocketTimeoutException e) { diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index c805627364..fedcfc97b9 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -18,15 +18,18 @@ import static java.lang.Math.max; import static java.nio.ByteBuffer.allocate; import static uk.ac.manchester.spinnaker.messages.Constants.UDP_MESSAGE_MAX_SIZE; +import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.read; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; +import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import uk.ac.manchester.spinnaker.connections.ConnectionSelector; import uk.ac.manchester.spinnaker.connections.SCPConnection; +import uk.ac.manchester.spinnaker.machine.ChipLocation; import uk.ac.manchester.spinnaker.machine.HasCoreLocation; import uk.ac.manchester.spinnaker.machine.MemoryLocation; import uk.ac.manchester.spinnaker.messages.scp.SendMCDataRequest; @@ -141,4 +144,54 @@ public void writeMemory( } finishBatch(); } + + /** + * Write to memory reading from a stream of data that includes context + * switches. Each section is a "header" of memory address (integer), + * chip x and y (each a short) and the number of words, followed by the + * words themselves. The number of words does not have to be short enough + * to fit in a packet; this will be managed by this function. + * + * @param core + * The location to send the messages to. + * @param data + * The stream of data to write. + * @throws IOException + * If anything goes wrong with networking or the input stream. + * @throws ProcessException + * If SpiNNaker rejects a message. + * @throws InterruptedException + * If the communications were interrupted. + */ + public void writeMemoryStream(HasCoreLocation core, InputStream data) + throws IOException, ProcessException, InterruptedException { + System.err.println("Reading information"); + DataInputStream input = new DataInputStream(data); + var baseAddress = new MemoryLocation(input.readInt()); + var targetChip = new ChipLocation(input.readShort(), input.readShort()); + var nWords = input.readInt(); + + System.err.println("Writing " + nWords + " words to " + baseAddress + + " on " + targetChip); + + var writePosition = baseAddress; + var nBytesRemaining = nWords * WORD_SIZE; + while (nBytesRemaining > 0) { + // One buffer per message; lifetime extends until batch end + System.err.println("Reading data; " + nBytesRemaining + " remaining"); + var tmp = read(input, allocate(UDP_MESSAGE_MAX_SIZE), + nBytesRemaining); + if (tmp == null) { + break; + } + System.err.println("Sending request"); + sendRequest(new SendMCDataRequest(core, targetChip, writePosition, + tmp)); + writePosition = writePosition.add(tmp.remaining()); + nBytesRemaining -= tmp.remaining(); + } + System.err.println("Finishing send"); + finishBatch(); + System.err.println("Finished!"); + } } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 8b0fae95d5..3e6ce67887 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -24,11 +24,14 @@ import static java.util.stream.Collectors.toList; import static org.slf4j.LoggerFactory.getLogger; import static uk.ac.manchester.spinnaker.messages.Constants.NBBY; +import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; +import static uk.ac.manchester.spinnaker.messages.Constants.UDP_MESSAGE_MAX_SIZE; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; import static uk.ac.manchester.spinnaker.utils.UnitConstants.MEGABYTE; import static uk.ac.manchester.spinnaker.utils.UnitConstants.NSEC_PER_SEC; import java.io.BufferedWriter; +import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -215,19 +218,20 @@ private void loadBoard(Ethernet board, DSEStorage storage) } log.info("loading data onto {} cores on board", cores.size()); var gather = gathererForChip.get(board.location); - var worker = new FastBoardWorker(txrx, board, storage, gather); - for (var xyp : cores) { - worker.mallocCore(xyp); - } - try (var routers = worker.systemRouterTables(); - var context = worker.dontDropPackets(gather)) { + try (var worker = new FastBoardWorker(txrx, board, storage, gather)) { for (var xyp : cores) { - worker.loadCore(xyp); + worker.mallocCore(xyp); + } + try (var routers = worker.systemRouterTables(); + var context = worker.dontDropPackets(gather)) { + for (var xyp : cores) { + worker.loadCore(xyp); + } + log.info("finished sending data in for this board"); + } catch (Exception e) { + log.warn("failure in core loading", e); + throw e; } - log.info("finished sending data in for this board"); - } catch (Exception e) { - log.warn("failure in core loading", e); - throw e; } } @@ -343,15 +347,45 @@ private static List describeChunk(Chunk chunk) { * @author Donal Fellows * @author Alan Stokes */ - private class FastBoardWorker extends BoardWorker { - private HasCoreLocation ethernet; + private class FastBoardWorker extends BoardWorker implements AutoCloseable { + + private final CoreLocation ethernet; + + private final PipedInputStream input; + + private final DataOutputStream output; + + private final Thread outputThread; + FastBoardWorker(TransceiverInterface txrx, Ethernet board, DSEStorage storage, Gather gather) throws IOException, ProcessException, InterruptedException, StorageException { super(txrx, board, storage); - this.ethernet = new CoreLocation(board.location, gather.getP()); + System.err.println("Making fast board worker for " + board); + ethernet = new CoreLocation(board.location, gather.getP()); + input = new PipedInputStream(); + output = new DataOutputStream(new PipedOutputStream(input)); + outputThread = new Thread(() -> { + while (true) { + try { + System.err.println(this + ": Write multicast stream..."); + txrx.writeMemoryMulticastStream(ethernet, input); + System.err.println(this + ": Finish write multicast stream"); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + outputThread.start(); + } + + @Override + public void close() throws IOException, InterruptedException { + output.close(); + outputThread.join(); + input.close(); } /** @@ -480,31 +514,31 @@ SystemRouterTableContext systemRouterTables() * If communications are interrupted. */ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, - ByteBuffer data) - throws IOException, InterruptedException { - - try (var input = new PipedInputStream(data.remaining()); - var output = new PipedOutputStream(input)) { - byte[] transfer = new byte[data.remaining()]; - data.get(transfer); - output.write(transfer); - output.flush(); - output.close(); - int boardLocalX = core.getX() - ethernet.getX(); - if (boardLocalX < 0) { - boardLocalX += machine.maxChipX() + 1; - } - int boardLocalY = core.getY() - ethernet.getY(); - if (boardLocalY < 0) { - boardLocalY += machine.maxChipY() + 1; - } - var boardLocal = new CoreLocation(boardLocalX, boardLocalY, - core.getP()); - txrx.writeMemoryMulticast(ethernet, boardLocal, baseAddress, - input); - } catch (ProcessException e) { - throw new IOException(e); + ByteBuffer content) + throws IOException, InterruptedException, ProcessException { + + int boardLocalX = core.getX() - ethernet.getX(); + if (boardLocalX < 0) { + boardLocalX += machine.maxChipX() + 1; } + int boardLocalY = core.getY() - ethernet.getY(); + if (boardLocalY < 0) { + boardLocalY += machine.maxChipY() + 1; + } + var data = content.duplicate(); + var nBytes = data.remaining(); + byte[] transfer = new byte[nBytes]; + data.get(transfer); + output.writeInt(baseAddress.address); + output.writeShort(boardLocalX); + output.writeShort(boardLocalY); + output.writeInt(nBytes / WORD_SIZE); + output.flush(); + System.err.println("Writing " + nBytes + " bytes to " + baseAddress); + output.write(transfer); + output.flush(); + System.err.println("Finished write"); + } } } From e55d9a75f713bd7c435f014bc8406949b223ecd0 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 09:25:43 +0000 Subject: [PATCH 14/27] Working multi-threaded --- .../spinnaker/transceiver/TxrxProcess.java | 19 +------- .../WriteMemoryByMulticastProcess.java | 8 ---- .../spinnaker/front_end/dse/BoardWorker.java | 17 +++++++ .../dse/FastMCExecuteDataSpecification.java | 48 ++++++++++--------- 4 files changed, 45 insertions(+), 47 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java index 8fffbde69f..e36e3b5216 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java @@ -303,16 +303,8 @@ protected TxrxProcess( * @return The pipeline instance. */ private RequestPipeline pipeline(SCPRequest request) { - System.err.println("Finding pipeline for request " + request); - var connection = selector.getNextConnection(request); - if (!requestPipelines.containsKey(connection)) { - System.err.println("Creating pipeline for connection " + connection); - var pipeline = new RequestPipeline(connection); - requestPipelines.put(connection, pipeline); - return pipeline; - } - System.err.println("Using existing pipeline for connection " + connection); - return requestPipelines.get(connection); + return requestPipelines.computeIfAbsent( + selector.getNextConnection(request), RequestPipeline::new); } /** @@ -337,7 +329,6 @@ private void resetFailureState() { protected final void finishBatch() throws ProcessException, IOException, InterruptedException { for (var pipe : requestPipelines.values()) { - System.err.println("Finishing pipe " + pipe); pipe.finish(); } if (failure != null) { @@ -575,7 +566,6 @@ public void send() throws IOException { } log.debug("Sending request {} with connection {}", request, connection); - System.err.println("Sending sequence " + seq + " with " + connection); switch (request.sdpHeader.getFlags()) { case REPLY_EXPECTED: case REPLY_EXPECTED_NO_P2P: @@ -826,7 +816,6 @@ private void multiRetrieve(int numPacketsOutstanding) // While there are still more packets in progress than some // threshold while (numOutstandingRequests() > numPacketsOutstanding) { - System.err.println("Waiting for " + numOutstandingRequests() + " requests of " + numPacketsOutstanding); try { // Receive the next response singleRetrieve(); @@ -840,7 +829,6 @@ private void singleRetrieve() throws IOException, InterruptedException { // Receive the next response log.debug("{}: Connection {} waiting for message... timeout of {}", this, connection, packetTimeout); - System.err.println("Waiting for message from connection " + connection); var msg = connection.receiveSCPResponse(packetTimeout); if (log.isDebugEnabled()) { log.debug( @@ -848,12 +836,10 @@ private void singleRetrieve() throws IOException, InterruptedException { this, connection, msg.getResult(), msg.getSequenceNumber()); } - System.err.println("Connection " + connection + " received message seq " + msg.getSequenceNumber()); var req = getRequestForResult(msg); // Only process responses which have matching requests if (req == null) { - System.err.println("Connection " + connection + " Unknown sequence!"); log.info("discarding message with unknown sequence number: {}", msg.getSequenceNumber()); if (log.isDebugEnabled()) { @@ -868,7 +854,6 @@ private void singleRetrieve() throws IOException, InterruptedException { // If the response can be retried, retry it if (msg.isRetriable()) { try { - System.err.println("Connection " + connection + " resending..."); resend(req, msg.getResult(), msg.getSequenceNumber()); numRetryCodeResent++; } catch (SocketTimeoutException e) { diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index fedcfc97b9..d2a0cf0535 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -165,33 +165,25 @@ public void writeMemory( */ public void writeMemoryStream(HasCoreLocation core, InputStream data) throws IOException, ProcessException, InterruptedException { - System.err.println("Reading information"); DataInputStream input = new DataInputStream(data); var baseAddress = new MemoryLocation(input.readInt()); var targetChip = new ChipLocation(input.readShort(), input.readShort()); var nWords = input.readInt(); - System.err.println("Writing " + nWords + " words to " + baseAddress - + " on " + targetChip); - var writePosition = baseAddress; var nBytesRemaining = nWords * WORD_SIZE; while (nBytesRemaining > 0) { // One buffer per message; lifetime extends until batch end - System.err.println("Reading data; " + nBytesRemaining + " remaining"); var tmp = read(input, allocate(UDP_MESSAGE_MAX_SIZE), nBytesRemaining); if (tmp == null) { break; } - System.err.println("Sending request"); sendRequest(new SendMCDataRequest(core, targetChip, writePosition, tmp)); writePosition = writePosition.add(tmp.remaining()); nBytesRemaining -= tmp.remaining(); } - System.err.println("Finishing send"); finishBatch(); - System.err.println("Finished!"); } } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/BoardWorker.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/BoardWorker.java index 65520d7439..e600ad0ca6 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/BoardWorker.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/BoardWorker.java @@ -199,6 +199,23 @@ protected void loadCore(CoreLocation xyp) throws IOException, var startAddress = storage.getStartAddress(xyp); pointerTable.flip(); + writePointerTable(xyp, startAddress, pointerTable); + } + + /** + * Do the writing of the pointer table. Separated to allow override e.g. + * if some of the writes above need to finish first! + * + * @param xyp The core to write the table to. + * @param startAddress The address to start the table at. + * @param pointerTable The table to write. + * @throws ProcessException If SCAMP rejects the request. + * @throws IOException If there is an I/O error. + * @throws InterruptedException If the process is interrupted. + */ + protected void writePointerTable(CoreLocation xyp, + MemoryLocation startAddress, ByteBuffer pointerTable) + throws ProcessException, IOException, InterruptedException { txrx.writeMemory(xyp.getScampCore(), startAddress, pointerTable); } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 3e6ce67887..0d33a5328c 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -25,13 +25,13 @@ import static org.slf4j.LoggerFactory.getLogger; import static uk.ac.manchester.spinnaker.messages.Constants.NBBY; import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; -import static uk.ac.manchester.spinnaker.messages.Constants.UDP_MESSAGE_MAX_SIZE; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; import static uk.ac.manchester.spinnaker.utils.UnitConstants.MEGABYTE; import static uk.ac.manchester.spinnaker.utils.UnitConstants.NSEC_PER_SEC; import java.io.BufferedWriter; import java.io.DataOutputStream; +import java.io.EOFException; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -218,20 +218,20 @@ private void loadBoard(Ethernet board, DSEStorage storage) } log.info("loading data onto {} cores on board", cores.size()); var gather = gathererForChip.get(board.location); - try (var worker = new FastBoardWorker(txrx, board, storage, gather)) { + var worker = new FastBoardWorker(txrx, board, storage, gather); + for (var xyp : cores) { + worker.mallocCore(xyp); + } + try (var routers = worker.systemRouterTables(); + var context = worker.dontDropPackets(gather)) { for (var xyp : cores) { - worker.mallocCore(xyp); - } - try (var routers = worker.systemRouterTables(); - var context = worker.dontDropPackets(gather)) { - for (var xyp : cores) { - worker.loadCore(xyp); - } - log.info("finished sending data in for this board"); - } catch (Exception e) { - log.warn("failure in core loading", e); - throw e; + worker.loadCore(xyp); } + worker.finishAll(); + log.info("finished sending data in for this board"); + } catch (Exception e) { + log.warn("failure in core loading", e); + throw e; } } @@ -347,7 +347,7 @@ private static List describeChunk(Chunk chunk) { * @author Donal Fellows * @author Alan Stokes */ - private class FastBoardWorker extends BoardWorker implements AutoCloseable { + private class FastBoardWorker extends BoardWorker { private final CoreLocation ethernet; @@ -363,16 +363,16 @@ private class FastBoardWorker extends BoardWorker implements AutoCloseable { throws IOException, ProcessException, InterruptedException, StorageException { super(txrx, board, storage); - System.err.println("Making fast board worker for " + board); ethernet = new CoreLocation(board.location, gather.getP()); input = new PipedInputStream(); output = new DataOutputStream(new PipedOutputStream(input)); outputThread = new Thread(() -> { - while (true) { + boolean finished = false; + while (!finished) { try { - System.err.println(this + ": Write multicast stream..."); txrx.writeMemoryMulticastStream(ethernet, input); - System.err.println(this + ": Finish write multicast stream"); + } catch (EOFException e) { + finished = true; } catch (Exception e) { e.printStackTrace(); } @@ -381,8 +381,7 @@ private class FastBoardWorker extends BoardWorker implements AutoCloseable { outputThread.start(); } - @Override - public void close() throws IOException, InterruptedException { + public void finishAll() throws IOException, InterruptedException { output.close(); outputThread.join(); input.close(); @@ -458,6 +457,13 @@ protected int writeRegion(HasCoreLocation core, ByteBuffer content, return written; } + @Override + protected void writePointerTable(CoreLocation xyp, + MemoryLocation startAddress, ByteBuffer pointerTable) + throws ProcessException, IOException, InterruptedException { + fastWrite(xyp, startAddress, pointerTable); + } + /** * Put the board in don't-drop-packets mode. * @@ -534,10 +540,8 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, output.writeShort(boardLocalY); output.writeInt(nBytes / WORD_SIZE); output.flush(); - System.err.println("Writing " + nBytes + " bytes to " + baseAddress); output.write(transfer); output.flush(); - System.err.println("Finished write"); } } From ceba9c7e2e74c1411440f8a96d4e6571a003980d Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 10:20:06 +0000 Subject: [PATCH 15/27] Working with gathering of packets! --- .../messages/scp/SendMCDataRequest.java | 17 ++++ .../WriteMemoryByMulticastProcess.java | 98 ++++++++++++++++--- .../dse/FastMCExecuteDataSpecification.java | 34 ++++--- 3 files changed, 120 insertions(+), 29 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java index 23431d2a17..237add22fa 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java @@ -54,6 +54,23 @@ public SendMCDataRequest(HasCoreLocation core, HasChipLocation targetChip, data.remaining() / WORD_SIZE, data); } + /** + * @param core + * Where to send the request. + * @param targetChip + * The target chip of the write. + * @param baseAddress + * The address to write to on the target core. + * @param data + * The data to write. + */ + public SendMCDataRequest(HasCoreLocation core, HasChipLocation targetChip, + MemoryLocation baseAddress, int nWords, ByteBuffer data) { + super(header(core), CMD_WRITE, baseAddress.address, + (targetChip.getX() << X_SHIFT) | targetChip.getY(), nWords, + data); + } + /** * Make a variant of SDP header that talks to the packet reinjector. It * always wants a reply and always talks to a particular SDP port diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index d2a0cf0535..9e8fc75d56 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -16,13 +16,16 @@ package uk.ac.manchester.spinnaker.transceiver; import static java.lang.Math.max; +import static java.lang.Math.min; import static java.nio.ByteBuffer.allocate; +import static java.nio.ByteOrder.LITTLE_ENDIAN; import static uk.ac.manchester.spinnaker.messages.Constants.UDP_MESSAGE_MAX_SIZE; import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.read; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; import java.io.DataInputStream; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -166,24 +169,91 @@ public void writeMemory( public void writeMemoryStream(HasCoreLocation core, InputStream data) throws IOException, ProcessException, InterruptedException { DataInputStream input = new DataInputStream(data); - var baseAddress = new MemoryLocation(input.readInt()); - var targetChip = new ChipLocation(input.readShort(), input.readShort()); - var nWords = input.readInt(); - var writePosition = baseAddress; - var nBytesRemaining = nWords * WORD_SIZE; - while (nBytesRemaining > 0) { - // One buffer per message; lifetime extends until batch end - var tmp = read(input, allocate(UDP_MESSAGE_MAX_SIZE), - nBytesRemaining); - if (tmp == null) { + // Keep a buffer to be sent, along with the initial send details + var nextBuffer = allocate(UDP_MESSAGE_MAX_SIZE).order(LITTLE_ENDIAN); + MemoryLocation nextAddress = null; + ChipLocation nextChip = null; + int nextNWords = -1; + + while (true) { + + // Read the next address and location to send to + MemoryLocation baseAddress; + try { + baseAddress = new MemoryLocation(input.readInt()); + } catch (EOFException e) { break; } - sendRequest(new SendMCDataRequest(core, targetChip, writePosition, - tmp)); - writePosition = writePosition.add(tmp.remaining()); - nBytesRemaining -= tmp.remaining(); + var targetChip = new ChipLocation(input.readShort(), + input.readShort()); + var nWords = input.readInt(); + + // Either we are at the start of the buffer, or there isn't enough + // space for the address info and some data (we need at least as + // much data as the header size to make it worth it) + if (nextBuffer.position() == 0 || + nextBuffer.remaining() < 2 * 3 * WORD_SIZE) { + + // If the buffer has some words in it, send it now + if (nextBuffer.position() > 0) { + nextBuffer.flip(); + sendRequest(new SendMCDataRequest(core, nextChip, + nextAddress, nextNWords, nextBuffer)); + nextBuffer = allocate(UDP_MESSAGE_MAX_SIZE) + .order(LITTLE_ENDIAN); + } + nextAddress = baseAddress; + nextChip = targetChip; + + // However the size depends on the space available! + nextNWords = min(nWords, UDP_MESSAGE_MAX_SIZE); + } else { + // We have enough space, so write the change of context to the + // buffer + nextBuffer.putInt(baseAddress.address); + nextBuffer.putShort((short) targetChip.getX()); + nextBuffer.putShort((short) targetChip.getY()); + + // Once again, the size depends on the space available! + nextBuffer.putInt(min(nWords, UDP_MESSAGE_MAX_SIZE)); + } + + var writePosition = baseAddress; + var nBytesRemaining = nWords * WORD_SIZE; + while (nBytesRemaining > 0) { + + // Read data into the buffer + var nextReadSize = min(nBytesRemaining, nextBuffer.remaining()); + input.read(nextBuffer.array(), nextBuffer.arrayOffset(), + nextReadSize); + + writePosition = writePosition.add(nextReadSize); + nBytesRemaining -= nextReadSize; + + // If the buffer is full, send it and update the header values + // for the potential next send + if (nextBuffer.remaining() == 0) { + nextBuffer.flip(); + sendRequest(new SendMCDataRequest(core, nextChip, + nextAddress, nextNWords, nextBuffer)); + nextBuffer = allocate(UDP_MESSAGE_MAX_SIZE) + .order(LITTLE_ENDIAN); + nextChip = targetChip; + nextAddress = writePosition; + nextNWords = min(nBytesRemaining / WORD_SIZE, + UDP_MESSAGE_MAX_SIZE); + } + } } + + // We might still have one last send to do... + if (nextBuffer.remaining() == 0) { + nextBuffer.flip(); + sendRequest(new SendMCDataRequest(core, nextChip, + nextAddress, nextNWords, nextBuffer)); + } + finishBatch(); } } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 0d33a5328c..56199ded96 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -357,6 +357,9 @@ private class FastBoardWorker extends BoardWorker { private final Thread outputThread; + private int totalDataWritten = 0; + + private long startTime = 0; FastBoardWorker(TransceiverInterface txrx, Ethernet board, DSEStorage storage, Gather gather) @@ -367,15 +370,12 @@ private class FastBoardWorker extends BoardWorker { input = new PipedInputStream(); output = new DataOutputStream(new PipedOutputStream(input)); outputThread = new Thread(() -> { - boolean finished = false; - while (!finished) { - try { - txrx.writeMemoryMulticastStream(ethernet, input); - } catch (EOFException e) { - finished = true; - } catch (Exception e) { - e.printStackTrace(); - } + try { + txrx.writeMemoryMulticastStream(ethernet, input); + } catch (EOFException e) { + // Ignore this + } catch (Exception e) { + e.printStackTrace(); } }); outputThread.start(); @@ -384,7 +384,12 @@ private class FastBoardWorker extends BoardWorker { public void finishAll() throws IOException, InterruptedException { output.close(); outputThread.join(); + var endTime = nanoTime(); input.close(); + + var recorder = new MissingRecorder(); + recorder.report(ethernet, endTime - startTime, totalDataWritten, + new MemoryLocation(0)); } /** @@ -443,12 +448,12 @@ void report(HasCoreLocation core, long time, int size, protected int writeRegion(HasCoreLocation core, ByteBuffer content, MemoryLocation baseAddress) throws IOException, ProcessException, InterruptedException { + if (startTime == 0) { + startTime = nanoTime(); + } int written = content.remaining(); - var recorder = new MissingRecorder(); - long start = nanoTime(); + totalDataWritten += written; fastWrite(core, baseAddress, content); - long end = nanoTime(); - recorder.report(core, end - start, content.limit(), baseAddress); if (SPINNAKER_COMPARE_UPLOAD != null) { var readBack = txrx.readMemory( core, baseAddress, content.remaining()); @@ -521,7 +526,7 @@ SystemRouterTableContext systemRouterTables() */ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, ByteBuffer content) - throws IOException, InterruptedException, ProcessException { + throws IOException, InterruptedException, ProcessException { int boardLocalX = core.getX() - ethernet.getX(); if (boardLocalX < 0) { @@ -542,7 +547,6 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, output.flush(); output.write(transfer); output.flush(); - } } } From 6039f1cfc95c2dba25aa71d726c4edb41eb9a668 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 12:24:51 +0000 Subject: [PATCH 16/27] Fixes --- .../WriteMemoryByMulticastProcess.java | 29 +++++++++++++------ .../dse/FastMCExecuteDataSpecification.java | 13 +++++++-- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index 9e8fc75d56..efd9425176 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -48,6 +48,9 @@ class WriteMemoryByMulticastProcess extends TxrxProcess { /** The number of simultaneous messages that can be in progress. */ private static final int N_CHANNELS = 8; + private static final int UDP_MESSAGE_MAX_WORDS = + UDP_MESSAGE_MAX_SIZE / WORD_SIZE; + /** * @param connectionSelector * How to select how to communicate. @@ -175,18 +178,21 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) MemoryLocation nextAddress = null; ChipLocation nextChip = null; int nextNWords = -1; + byte[] buffer = new byte[UDP_MESSAGE_MAX_SIZE]; while (true) { // Read the next address and location to send to MemoryLocation baseAddress; try { - baseAddress = new MemoryLocation(input.readInt()); + int addr = input.readInt(); + baseAddress = new MemoryLocation(addr); } catch (EOFException e) { break; } - var targetChip = new ChipLocation(input.readShort(), - input.readShort()); + int x = input.readShort(); + int y = input.readShort(); + var targetChip = new ChipLocation(x, y); var nWords = input.readInt(); // Either we are at the start of the buffer, or there isn't enough @@ -207,16 +213,17 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) nextChip = targetChip; // However the size depends on the space available! - nextNWords = min(nWords, UDP_MESSAGE_MAX_SIZE); + nextNWords = min(nWords, UDP_MESSAGE_MAX_WORDS); } else { // We have enough space, so write the change of context to the // buffer + var nWordsAvailable = (nextBuffer.remaining() / WORD_SIZE) - 3; nextBuffer.putInt(baseAddress.address); - nextBuffer.putShort((short) targetChip.getX()); nextBuffer.putShort((short) targetChip.getY()); + nextBuffer.putShort((short) targetChip.getX()); // Once again, the size depends on the space available! - nextBuffer.putInt(min(nWords, UDP_MESSAGE_MAX_SIZE)); + nextBuffer.putInt(min(nWords, nWordsAvailable)); } var writePosition = baseAddress; @@ -224,9 +231,13 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) while (nBytesRemaining > 0) { // Read data into the buffer + if ((nBytesRemaining / WORD_SIZE) * WORD_SIZE != nBytesRemaining) { + throw new IOException("Remaining bytes " + nBytesRemaining + + " is not a multiple of word size " + WORD_SIZE); + } var nextReadSize = min(nBytesRemaining, nextBuffer.remaining()); - input.read(nextBuffer.array(), nextBuffer.arrayOffset(), - nextReadSize); + input.readFully(buffer, 0, nextReadSize); + nextBuffer.put(buffer, 0, nextReadSize); writePosition = writePosition.add(nextReadSize); nBytesRemaining -= nextReadSize; @@ -242,7 +253,7 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) nextChip = targetChip; nextAddress = writePosition; nextNWords = min(nBytesRemaining / WORD_SIZE, - UDP_MESSAGE_MAX_SIZE); + UDP_MESSAGE_MAX_WORDS); } } } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 56199ded96..9f587f2770 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -357,6 +357,8 @@ private class FastBoardWorker extends BoardWorker { private final Thread outputThread; + private Exception exception = null; + private int totalDataWritten = 0; private long startTime = 0; @@ -375,7 +377,7 @@ private class FastBoardWorker extends BoardWorker { } catch (EOFException e) { // Ignore this } catch (Exception e) { - e.printStackTrace(); + exception = e; } }); outputThread.start(); @@ -387,6 +389,10 @@ public void finishAll() throws IOException, InterruptedException { var endTime = nanoTime(); input.close(); + if (exception != null) { + throw new IOException(exception); + } + var recorder = new MissingRecorder(); recorder.report(ethernet, endTime - startTime, totalDataWritten, new MemoryLocation(0)); @@ -400,7 +406,6 @@ public void finishAll() throws IOException, InterruptedException { */ private final class MissingRecorder { - /** * Issue the report based on what we recorded, if appropriate. * @@ -538,6 +543,10 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, } var data = content.duplicate(); var nBytes = data.remaining(); + if ((nBytes / WORD_SIZE) * WORD_SIZE != nBytes) { + throw new IOException("Data with " + nBytes + " is not a " + + "multiple of " + WORD_SIZE); + } byte[] transfer = new byte[nBytes]; data.get(transfer); output.writeInt(baseAddress.address); From c16ec7284e2e39cc0e8ddd7605e1d46fb04cd084 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 14:18:40 +0000 Subject: [PATCH 17/27] Write the last packet (doh) --- .../spinnaker/transceiver/WriteMemoryByMulticastProcess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index efd9425176..10ef4b8bb5 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -259,7 +259,7 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) } // We might still have one last send to do... - if (nextBuffer.remaining() == 0) { + if (nextBuffer.remaining() > 0) { nextBuffer.flip(); sendRequest(new SendMCDataRequest(core, nextChip, nextAddress, nextNWords, nextBuffer)); From 9f44da8c7dba2506c0a2071fd9b2e83ac1ebe3d6 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 14:27:08 +0000 Subject: [PATCH 18/27] Revert "Write the last packet (doh)" This reverts commit c16ec7284e2e39cc0e8ddd7605e1d46fb04cd084. --- .../spinnaker/transceiver/WriteMemoryByMulticastProcess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index 10ef4b8bb5..efd9425176 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -259,7 +259,7 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) } // We might still have one last send to do... - if (nextBuffer.remaining() > 0) { + if (nextBuffer.remaining() == 0) { nextBuffer.flip(); sendRequest(new SendMCDataRequest(core, nextChip, nextAddress, nextNWords, nextBuffer)); From 3c5844ae522c5a54acd9963a3d7b50864eff9945 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 14:27:09 +0000 Subject: [PATCH 19/27] Revert "Fixes" This reverts commit 6039f1cfc95c2dba25aa71d726c4edb41eb9a668. --- .../WriteMemoryByMulticastProcess.java | 29 ++++++------------- .../dse/FastMCExecuteDataSpecification.java | 13 ++------- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index efd9425176..9e8fc75d56 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -48,9 +48,6 @@ class WriteMemoryByMulticastProcess extends TxrxProcess { /** The number of simultaneous messages that can be in progress. */ private static final int N_CHANNELS = 8; - private static final int UDP_MESSAGE_MAX_WORDS = - UDP_MESSAGE_MAX_SIZE / WORD_SIZE; - /** * @param connectionSelector * How to select how to communicate. @@ -178,21 +175,18 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) MemoryLocation nextAddress = null; ChipLocation nextChip = null; int nextNWords = -1; - byte[] buffer = new byte[UDP_MESSAGE_MAX_SIZE]; while (true) { // Read the next address and location to send to MemoryLocation baseAddress; try { - int addr = input.readInt(); - baseAddress = new MemoryLocation(addr); + baseAddress = new MemoryLocation(input.readInt()); } catch (EOFException e) { break; } - int x = input.readShort(); - int y = input.readShort(); - var targetChip = new ChipLocation(x, y); + var targetChip = new ChipLocation(input.readShort(), + input.readShort()); var nWords = input.readInt(); // Either we are at the start of the buffer, or there isn't enough @@ -213,17 +207,16 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) nextChip = targetChip; // However the size depends on the space available! - nextNWords = min(nWords, UDP_MESSAGE_MAX_WORDS); + nextNWords = min(nWords, UDP_MESSAGE_MAX_SIZE); } else { // We have enough space, so write the change of context to the // buffer - var nWordsAvailable = (nextBuffer.remaining() / WORD_SIZE) - 3; nextBuffer.putInt(baseAddress.address); - nextBuffer.putShort((short) targetChip.getY()); nextBuffer.putShort((short) targetChip.getX()); + nextBuffer.putShort((short) targetChip.getY()); // Once again, the size depends on the space available! - nextBuffer.putInt(min(nWords, nWordsAvailable)); + nextBuffer.putInt(min(nWords, UDP_MESSAGE_MAX_SIZE)); } var writePosition = baseAddress; @@ -231,13 +224,9 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) while (nBytesRemaining > 0) { // Read data into the buffer - if ((nBytesRemaining / WORD_SIZE) * WORD_SIZE != nBytesRemaining) { - throw new IOException("Remaining bytes " + nBytesRemaining - + " is not a multiple of word size " + WORD_SIZE); - } var nextReadSize = min(nBytesRemaining, nextBuffer.remaining()); - input.readFully(buffer, 0, nextReadSize); - nextBuffer.put(buffer, 0, nextReadSize); + input.read(nextBuffer.array(), nextBuffer.arrayOffset(), + nextReadSize); writePosition = writePosition.add(nextReadSize); nBytesRemaining -= nextReadSize; @@ -253,7 +242,7 @@ public void writeMemoryStream(HasCoreLocation core, InputStream data) nextChip = targetChip; nextAddress = writePosition; nextNWords = min(nBytesRemaining / WORD_SIZE, - UDP_MESSAGE_MAX_WORDS); + UDP_MESSAGE_MAX_SIZE); } } } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 9f587f2770..56199ded96 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -357,8 +357,6 @@ private class FastBoardWorker extends BoardWorker { private final Thread outputThread; - private Exception exception = null; - private int totalDataWritten = 0; private long startTime = 0; @@ -377,7 +375,7 @@ private class FastBoardWorker extends BoardWorker { } catch (EOFException e) { // Ignore this } catch (Exception e) { - exception = e; + e.printStackTrace(); } }); outputThread.start(); @@ -389,10 +387,6 @@ public void finishAll() throws IOException, InterruptedException { var endTime = nanoTime(); input.close(); - if (exception != null) { - throw new IOException(exception); - } - var recorder = new MissingRecorder(); recorder.report(ethernet, endTime - startTime, totalDataWritten, new MemoryLocation(0)); @@ -406,6 +400,7 @@ public void finishAll() throws IOException, InterruptedException { */ private final class MissingRecorder { + /** * Issue the report based on what we recorded, if appropriate. * @@ -543,10 +538,6 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, } var data = content.duplicate(); var nBytes = data.remaining(); - if ((nBytes / WORD_SIZE) * WORD_SIZE != nBytes) { - throw new IOException("Data with " + nBytes + " is not a " - + "multiple of " + WORD_SIZE); - } byte[] transfer = new byte[nBytes]; data.get(transfer); output.writeInt(baseAddress.address); From 8c1693cbf2ecc14de37a92f62f231b7d3fd6bd89 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 14:27:09 +0000 Subject: [PATCH 20/27] Revert "Working with gathering of packets!" This reverts commit ceba9c7e2e74c1411440f8a96d4e6571a003980d. --- .../messages/scp/SendMCDataRequest.java | 17 ---- .../WriteMemoryByMulticastProcess.java | 98 +++---------------- .../dse/FastMCExecuteDataSpecification.java | 34 +++---- 3 files changed, 29 insertions(+), 120 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java index 237add22fa..23431d2a17 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java @@ -54,23 +54,6 @@ public SendMCDataRequest(HasCoreLocation core, HasChipLocation targetChip, data.remaining() / WORD_SIZE, data); } - /** - * @param core - * Where to send the request. - * @param targetChip - * The target chip of the write. - * @param baseAddress - * The address to write to on the target core. - * @param data - * The data to write. - */ - public SendMCDataRequest(HasCoreLocation core, HasChipLocation targetChip, - MemoryLocation baseAddress, int nWords, ByteBuffer data) { - super(header(core), CMD_WRITE, baseAddress.address, - (targetChip.getX() << X_SHIFT) | targetChip.getY(), nWords, - data); - } - /** * Make a variant of SDP header that talks to the packet reinjector. It * always wants a reply and always talks to a particular SDP port diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index 9e8fc75d56..d2a0cf0535 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -16,16 +16,13 @@ package uk.ac.manchester.spinnaker.transceiver; import static java.lang.Math.max; -import static java.lang.Math.min; import static java.nio.ByteBuffer.allocate; -import static java.nio.ByteOrder.LITTLE_ENDIAN; import static uk.ac.manchester.spinnaker.messages.Constants.UDP_MESSAGE_MAX_SIZE; import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.read; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; import java.io.DataInputStream; -import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -169,91 +166,24 @@ public void writeMemory( public void writeMemoryStream(HasCoreLocation core, InputStream data) throws IOException, ProcessException, InterruptedException { DataInputStream input = new DataInputStream(data); + var baseAddress = new MemoryLocation(input.readInt()); + var targetChip = new ChipLocation(input.readShort(), input.readShort()); + var nWords = input.readInt(); - // Keep a buffer to be sent, along with the initial send details - var nextBuffer = allocate(UDP_MESSAGE_MAX_SIZE).order(LITTLE_ENDIAN); - MemoryLocation nextAddress = null; - ChipLocation nextChip = null; - int nextNWords = -1; - - while (true) { - - // Read the next address and location to send to - MemoryLocation baseAddress; - try { - baseAddress = new MemoryLocation(input.readInt()); - } catch (EOFException e) { + var writePosition = baseAddress; + var nBytesRemaining = nWords * WORD_SIZE; + while (nBytesRemaining > 0) { + // One buffer per message; lifetime extends until batch end + var tmp = read(input, allocate(UDP_MESSAGE_MAX_SIZE), + nBytesRemaining); + if (tmp == null) { break; } - var targetChip = new ChipLocation(input.readShort(), - input.readShort()); - var nWords = input.readInt(); - - // Either we are at the start of the buffer, or there isn't enough - // space for the address info and some data (we need at least as - // much data as the header size to make it worth it) - if (nextBuffer.position() == 0 || - nextBuffer.remaining() < 2 * 3 * WORD_SIZE) { - - // If the buffer has some words in it, send it now - if (nextBuffer.position() > 0) { - nextBuffer.flip(); - sendRequest(new SendMCDataRequest(core, nextChip, - nextAddress, nextNWords, nextBuffer)); - nextBuffer = allocate(UDP_MESSAGE_MAX_SIZE) - .order(LITTLE_ENDIAN); - } - nextAddress = baseAddress; - nextChip = targetChip; - - // However the size depends on the space available! - nextNWords = min(nWords, UDP_MESSAGE_MAX_SIZE); - } else { - // We have enough space, so write the change of context to the - // buffer - nextBuffer.putInt(baseAddress.address); - nextBuffer.putShort((short) targetChip.getX()); - nextBuffer.putShort((short) targetChip.getY()); - - // Once again, the size depends on the space available! - nextBuffer.putInt(min(nWords, UDP_MESSAGE_MAX_SIZE)); - } - - var writePosition = baseAddress; - var nBytesRemaining = nWords * WORD_SIZE; - while (nBytesRemaining > 0) { - - // Read data into the buffer - var nextReadSize = min(nBytesRemaining, nextBuffer.remaining()); - input.read(nextBuffer.array(), nextBuffer.arrayOffset(), - nextReadSize); - - writePosition = writePosition.add(nextReadSize); - nBytesRemaining -= nextReadSize; - - // If the buffer is full, send it and update the header values - // for the potential next send - if (nextBuffer.remaining() == 0) { - nextBuffer.flip(); - sendRequest(new SendMCDataRequest(core, nextChip, - nextAddress, nextNWords, nextBuffer)); - nextBuffer = allocate(UDP_MESSAGE_MAX_SIZE) - .order(LITTLE_ENDIAN); - nextChip = targetChip; - nextAddress = writePosition; - nextNWords = min(nBytesRemaining / WORD_SIZE, - UDP_MESSAGE_MAX_SIZE); - } - } - } - - // We might still have one last send to do... - if (nextBuffer.remaining() == 0) { - nextBuffer.flip(); - sendRequest(new SendMCDataRequest(core, nextChip, - nextAddress, nextNWords, nextBuffer)); + sendRequest(new SendMCDataRequest(core, targetChip, writePosition, + tmp)); + writePosition = writePosition.add(tmp.remaining()); + nBytesRemaining -= tmp.remaining(); } - finishBatch(); } } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 56199ded96..0d33a5328c 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -357,9 +357,6 @@ private class FastBoardWorker extends BoardWorker { private final Thread outputThread; - private int totalDataWritten = 0; - - private long startTime = 0; FastBoardWorker(TransceiverInterface txrx, Ethernet board, DSEStorage storage, Gather gather) @@ -370,12 +367,15 @@ private class FastBoardWorker extends BoardWorker { input = new PipedInputStream(); output = new DataOutputStream(new PipedOutputStream(input)); outputThread = new Thread(() -> { - try { - txrx.writeMemoryMulticastStream(ethernet, input); - } catch (EOFException e) { - // Ignore this - } catch (Exception e) { - e.printStackTrace(); + boolean finished = false; + while (!finished) { + try { + txrx.writeMemoryMulticastStream(ethernet, input); + } catch (EOFException e) { + finished = true; + } catch (Exception e) { + e.printStackTrace(); + } } }); outputThread.start(); @@ -384,12 +384,7 @@ private class FastBoardWorker extends BoardWorker { public void finishAll() throws IOException, InterruptedException { output.close(); outputThread.join(); - var endTime = nanoTime(); input.close(); - - var recorder = new MissingRecorder(); - recorder.report(ethernet, endTime - startTime, totalDataWritten, - new MemoryLocation(0)); } /** @@ -448,12 +443,12 @@ void report(HasCoreLocation core, long time, int size, protected int writeRegion(HasCoreLocation core, ByteBuffer content, MemoryLocation baseAddress) throws IOException, ProcessException, InterruptedException { - if (startTime == 0) { - startTime = nanoTime(); - } int written = content.remaining(); - totalDataWritten += written; + var recorder = new MissingRecorder(); + long start = nanoTime(); fastWrite(core, baseAddress, content); + long end = nanoTime(); + recorder.report(core, end - start, content.limit(), baseAddress); if (SPINNAKER_COMPARE_UPLOAD != null) { var readBack = txrx.readMemory( core, baseAddress, content.remaining()); @@ -526,7 +521,7 @@ SystemRouterTableContext systemRouterTables() */ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, ByteBuffer content) - throws IOException, InterruptedException, ProcessException { + throws IOException, InterruptedException, ProcessException { int boardLocalX = core.getX() - ethernet.getX(); if (boardLocalX < 0) { @@ -547,6 +542,7 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, output.flush(); output.write(transfer); output.flush(); + } } } From 746a521a70e5e411d8f0359777c29f403c473b3e Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 14:27:09 +0000 Subject: [PATCH 21/27] Revert "Working multi-threaded" This reverts commit e55d9a75f713bd7c435f014bc8406949b223ecd0. --- .../spinnaker/transceiver/TxrxProcess.java | 19 +++++++- .../WriteMemoryByMulticastProcess.java | 8 ++++ .../spinnaker/front_end/dse/BoardWorker.java | 17 ------- .../dse/FastMCExecuteDataSpecification.java | 48 +++++++++---------- 4 files changed, 47 insertions(+), 45 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java index e36e3b5216..8fffbde69f 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java @@ -303,8 +303,16 @@ protected TxrxProcess( * @return The pipeline instance. */ private RequestPipeline pipeline(SCPRequest request) { - return requestPipelines.computeIfAbsent( - selector.getNextConnection(request), RequestPipeline::new); + System.err.println("Finding pipeline for request " + request); + var connection = selector.getNextConnection(request); + if (!requestPipelines.containsKey(connection)) { + System.err.println("Creating pipeline for connection " + connection); + var pipeline = new RequestPipeline(connection); + requestPipelines.put(connection, pipeline); + return pipeline; + } + System.err.println("Using existing pipeline for connection " + connection); + return requestPipelines.get(connection); } /** @@ -329,6 +337,7 @@ private void resetFailureState() { protected final void finishBatch() throws ProcessException, IOException, InterruptedException { for (var pipe : requestPipelines.values()) { + System.err.println("Finishing pipe " + pipe); pipe.finish(); } if (failure != null) { @@ -566,6 +575,7 @@ public void send() throws IOException { } log.debug("Sending request {} with connection {}", request, connection); + System.err.println("Sending sequence " + seq + " with " + connection); switch (request.sdpHeader.getFlags()) { case REPLY_EXPECTED: case REPLY_EXPECTED_NO_P2P: @@ -816,6 +826,7 @@ private void multiRetrieve(int numPacketsOutstanding) // While there are still more packets in progress than some // threshold while (numOutstandingRequests() > numPacketsOutstanding) { + System.err.println("Waiting for " + numOutstandingRequests() + " requests of " + numPacketsOutstanding); try { // Receive the next response singleRetrieve(); @@ -829,6 +840,7 @@ private void singleRetrieve() throws IOException, InterruptedException { // Receive the next response log.debug("{}: Connection {} waiting for message... timeout of {}", this, connection, packetTimeout); + System.err.println("Waiting for message from connection " + connection); var msg = connection.receiveSCPResponse(packetTimeout); if (log.isDebugEnabled()) { log.debug( @@ -836,10 +848,12 @@ private void singleRetrieve() throws IOException, InterruptedException { this, connection, msg.getResult(), msg.getSequenceNumber()); } + System.err.println("Connection " + connection + " received message seq " + msg.getSequenceNumber()); var req = getRequestForResult(msg); // Only process responses which have matching requests if (req == null) { + System.err.println("Connection " + connection + " Unknown sequence!"); log.info("discarding message with unknown sequence number: {}", msg.getSequenceNumber()); if (log.isDebugEnabled()) { @@ -854,6 +868,7 @@ private void singleRetrieve() throws IOException, InterruptedException { // If the response can be retried, retry it if (msg.isRetriable()) { try { + System.err.println("Connection " + connection + " resending..."); resend(req, msg.getResult(), msg.getSequenceNumber()); numRetryCodeResent++; } catch (SocketTimeoutException e) { diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index d2a0cf0535..fedcfc97b9 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -165,25 +165,33 @@ public void writeMemory( */ public void writeMemoryStream(HasCoreLocation core, InputStream data) throws IOException, ProcessException, InterruptedException { + System.err.println("Reading information"); DataInputStream input = new DataInputStream(data); var baseAddress = new MemoryLocation(input.readInt()); var targetChip = new ChipLocation(input.readShort(), input.readShort()); var nWords = input.readInt(); + System.err.println("Writing " + nWords + " words to " + baseAddress + + " on " + targetChip); + var writePosition = baseAddress; var nBytesRemaining = nWords * WORD_SIZE; while (nBytesRemaining > 0) { // One buffer per message; lifetime extends until batch end + System.err.println("Reading data; " + nBytesRemaining + " remaining"); var tmp = read(input, allocate(UDP_MESSAGE_MAX_SIZE), nBytesRemaining); if (tmp == null) { break; } + System.err.println("Sending request"); sendRequest(new SendMCDataRequest(core, targetChip, writePosition, tmp)); writePosition = writePosition.add(tmp.remaining()); nBytesRemaining -= tmp.remaining(); } + System.err.println("Finishing send"); finishBatch(); + System.err.println("Finished!"); } } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/BoardWorker.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/BoardWorker.java index e600ad0ca6..65520d7439 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/BoardWorker.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/BoardWorker.java @@ -199,23 +199,6 @@ protected void loadCore(CoreLocation xyp) throws IOException, var startAddress = storage.getStartAddress(xyp); pointerTable.flip(); - writePointerTable(xyp, startAddress, pointerTable); - } - - /** - * Do the writing of the pointer table. Separated to allow override e.g. - * if some of the writes above need to finish first! - * - * @param xyp The core to write the table to. - * @param startAddress The address to start the table at. - * @param pointerTable The table to write. - * @throws ProcessException If SCAMP rejects the request. - * @throws IOException If there is an I/O error. - * @throws InterruptedException If the process is interrupted. - */ - protected void writePointerTable(CoreLocation xyp, - MemoryLocation startAddress, ByteBuffer pointerTable) - throws ProcessException, IOException, InterruptedException { txrx.writeMemory(xyp.getScampCore(), startAddress, pointerTable); } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 0d33a5328c..3e6ce67887 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -25,13 +25,13 @@ import static org.slf4j.LoggerFactory.getLogger; import static uk.ac.manchester.spinnaker.messages.Constants.NBBY; import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; +import static uk.ac.manchester.spinnaker.messages.Constants.UDP_MESSAGE_MAX_SIZE; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; import static uk.ac.manchester.spinnaker.utils.UnitConstants.MEGABYTE; import static uk.ac.manchester.spinnaker.utils.UnitConstants.NSEC_PER_SEC; import java.io.BufferedWriter; import java.io.DataOutputStream; -import java.io.EOFException; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -218,20 +218,20 @@ private void loadBoard(Ethernet board, DSEStorage storage) } log.info("loading data onto {} cores on board", cores.size()); var gather = gathererForChip.get(board.location); - var worker = new FastBoardWorker(txrx, board, storage, gather); - for (var xyp : cores) { - worker.mallocCore(xyp); - } - try (var routers = worker.systemRouterTables(); - var context = worker.dontDropPackets(gather)) { + try (var worker = new FastBoardWorker(txrx, board, storage, gather)) { for (var xyp : cores) { - worker.loadCore(xyp); + worker.mallocCore(xyp); + } + try (var routers = worker.systemRouterTables(); + var context = worker.dontDropPackets(gather)) { + for (var xyp : cores) { + worker.loadCore(xyp); + } + log.info("finished sending data in for this board"); + } catch (Exception e) { + log.warn("failure in core loading", e); + throw e; } - worker.finishAll(); - log.info("finished sending data in for this board"); - } catch (Exception e) { - log.warn("failure in core loading", e); - throw e; } } @@ -347,7 +347,7 @@ private static List describeChunk(Chunk chunk) { * @author Donal Fellows * @author Alan Stokes */ - private class FastBoardWorker extends BoardWorker { + private class FastBoardWorker extends BoardWorker implements AutoCloseable { private final CoreLocation ethernet; @@ -363,16 +363,16 @@ private class FastBoardWorker extends BoardWorker { throws IOException, ProcessException, InterruptedException, StorageException { super(txrx, board, storage); + System.err.println("Making fast board worker for " + board); ethernet = new CoreLocation(board.location, gather.getP()); input = new PipedInputStream(); output = new DataOutputStream(new PipedOutputStream(input)); outputThread = new Thread(() -> { - boolean finished = false; - while (!finished) { + while (true) { try { + System.err.println(this + ": Write multicast stream..."); txrx.writeMemoryMulticastStream(ethernet, input); - } catch (EOFException e) { - finished = true; + System.err.println(this + ": Finish write multicast stream"); } catch (Exception e) { e.printStackTrace(); } @@ -381,7 +381,8 @@ private class FastBoardWorker extends BoardWorker { outputThread.start(); } - public void finishAll() throws IOException, InterruptedException { + @Override + public void close() throws IOException, InterruptedException { output.close(); outputThread.join(); input.close(); @@ -457,13 +458,6 @@ protected int writeRegion(HasCoreLocation core, ByteBuffer content, return written; } - @Override - protected void writePointerTable(CoreLocation xyp, - MemoryLocation startAddress, ByteBuffer pointerTable) - throws ProcessException, IOException, InterruptedException { - fastWrite(xyp, startAddress, pointerTable); - } - /** * Put the board in don't-drop-packets mode. * @@ -540,8 +534,10 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, output.writeShort(boardLocalY); output.writeInt(nBytes / WORD_SIZE); output.flush(); + System.err.println("Writing " + nBytes + " bytes to " + baseAddress); output.write(transfer); output.flush(); + System.err.println("Finished write"); } } From 3835ecbed92722a04eb02234bc5f4f524d2c365f Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 14:27:09 +0000 Subject: [PATCH 22/27] Revert "Debugging... need thread-safe multi-channel system!" This reverts commit 9b43acf096eb36f3f355a316a20071bb80051f0d. --- .../messages/scp/SendMCDataRequest.java | 9 +- .../spinnaker/transceiver/Transceiver.java | 9 -- .../transceiver/TransceiverInterface.java | 21 ---- .../spinnaker/transceiver/TxrxProcess.java | 19 +-- .../WriteMemoryByMulticastProcess.java | 53 --------- .../dse/FastMCExecuteDataSpecification.java | 110 ++++++------------ 6 files changed, 44 insertions(+), 177 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java index 23431d2a17..8279601023 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/messages/scp/SendMCDataRequest.java @@ -23,7 +23,6 @@ import java.nio.ByteBuffer; -import uk.ac.manchester.spinnaker.machine.HasChipLocation; import uk.ac.manchester.spinnaker.machine.HasCoreLocation; import uk.ac.manchester.spinnaker.machine.MemoryLocation; import uk.ac.manchester.spinnaker.messages.model.UnexpectedResponseCodeException; @@ -40,17 +39,17 @@ public class SendMCDataRequest extends SCPRequest { /** * @param core * Where to send the request. - * @param targetChip - * The target chip of the write. + * @param targetCore + * The target core of the write. * @param baseAddress * The address to write to on the target core. * @param data * The data to write. */ - public SendMCDataRequest(HasCoreLocation core, HasChipLocation targetChip, + public SendMCDataRequest(HasCoreLocation core, HasCoreLocation targetCore, MemoryLocation baseAddress, ByteBuffer data) { super(header(core), CMD_WRITE, baseAddress.address, - (targetChip.getX() << X_SHIFT) | targetChip.getY(), + (targetCore.getX() << X_SHIFT) | targetCore.getY(), data.remaining() / WORD_SIZE, data); } diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java index 7596e32feb..e61ec6b4bd 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java @@ -2106,15 +2106,6 @@ public void writeMemoryMulticast(HasCoreLocation core, core, targetCore, baseAddress, data); } - @Override - @ParallelSafe - public void writeMemoryMulticastStream(HasCoreLocation core, - InputStream data) - throws IOException, ProcessException, InterruptedException { - new WriteMemoryByMulticastProcess(scpSelector, this).writeMemoryStream( - core, data); - } - @Override @CheckReturnValue @ParallelSafe diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java index 2c7769e16c..95f5cb4f29 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java @@ -1921,27 +1921,6 @@ void writeMemoryMulticast(@Valid HasCoreLocation core, @NotNull MemoryLocation baseAddress, @NotNull InputStream data) throws IOException, ProcessException, InterruptedException; - /** - * Write to the board via multicast on the Ethernet chip from a stream which - * includes headers to indicate where data is to be streamed. - * - * @param core - * The coordinates of the Ethernet core containing the advanced - * monitor support - * @param data - * The stream of data to be written. - * @throws IOException - * If anything goes wrong with networking. - * @throws ProcessException - * If SpiNNaker rejects a message. - * @throws InterruptedException - * If the communications were interrupted. - */ - @ParallelSafe - void writeMemoryMulticastStream(@Valid HasCoreLocation core, - @NotNull InputStream data) - throws IOException, ProcessException, InterruptedException; - /** * Write to the user0 register of a core. * diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java index 8fffbde69f..e36e3b5216 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TxrxProcess.java @@ -303,16 +303,8 @@ protected TxrxProcess( * @return The pipeline instance. */ private RequestPipeline pipeline(SCPRequest request) { - System.err.println("Finding pipeline for request " + request); - var connection = selector.getNextConnection(request); - if (!requestPipelines.containsKey(connection)) { - System.err.println("Creating pipeline for connection " + connection); - var pipeline = new RequestPipeline(connection); - requestPipelines.put(connection, pipeline); - return pipeline; - } - System.err.println("Using existing pipeline for connection " + connection); - return requestPipelines.get(connection); + return requestPipelines.computeIfAbsent( + selector.getNextConnection(request), RequestPipeline::new); } /** @@ -337,7 +329,6 @@ private void resetFailureState() { protected final void finishBatch() throws ProcessException, IOException, InterruptedException { for (var pipe : requestPipelines.values()) { - System.err.println("Finishing pipe " + pipe); pipe.finish(); } if (failure != null) { @@ -575,7 +566,6 @@ public void send() throws IOException { } log.debug("Sending request {} with connection {}", request, connection); - System.err.println("Sending sequence " + seq + " with " + connection); switch (request.sdpHeader.getFlags()) { case REPLY_EXPECTED: case REPLY_EXPECTED_NO_P2P: @@ -826,7 +816,6 @@ private void multiRetrieve(int numPacketsOutstanding) // While there are still more packets in progress than some // threshold while (numOutstandingRequests() > numPacketsOutstanding) { - System.err.println("Waiting for " + numOutstandingRequests() + " requests of " + numPacketsOutstanding); try { // Receive the next response singleRetrieve(); @@ -840,7 +829,6 @@ private void singleRetrieve() throws IOException, InterruptedException { // Receive the next response log.debug("{}: Connection {} waiting for message... timeout of {}", this, connection, packetTimeout); - System.err.println("Waiting for message from connection " + connection); var msg = connection.receiveSCPResponse(packetTimeout); if (log.isDebugEnabled()) { log.debug( @@ -848,12 +836,10 @@ private void singleRetrieve() throws IOException, InterruptedException { this, connection, msg.getResult(), msg.getSequenceNumber()); } - System.err.println("Connection " + connection + " received message seq " + msg.getSequenceNumber()); var req = getRequestForResult(msg); // Only process responses which have matching requests if (req == null) { - System.err.println("Connection " + connection + " Unknown sequence!"); log.info("discarding message with unknown sequence number: {}", msg.getSequenceNumber()); if (log.isDebugEnabled()) { @@ -868,7 +854,6 @@ private void singleRetrieve() throws IOException, InterruptedException { // If the response can be retried, retry it if (msg.isRetriable()) { try { - System.err.println("Connection " + connection + " resending..."); resend(req, msg.getResult(), msg.getSequenceNumber()); numRetryCodeResent++; } catch (SocketTimeoutException e) { diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java index fedcfc97b9..c805627364 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/WriteMemoryByMulticastProcess.java @@ -18,18 +18,15 @@ import static java.lang.Math.max; import static java.nio.ByteBuffer.allocate; import static uk.ac.manchester.spinnaker.messages.Constants.UDP_MESSAGE_MAX_SIZE; -import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.read; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; -import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import uk.ac.manchester.spinnaker.connections.ConnectionSelector; import uk.ac.manchester.spinnaker.connections.SCPConnection; -import uk.ac.manchester.spinnaker.machine.ChipLocation; import uk.ac.manchester.spinnaker.machine.HasCoreLocation; import uk.ac.manchester.spinnaker.machine.MemoryLocation; import uk.ac.manchester.spinnaker.messages.scp.SendMCDataRequest; @@ -144,54 +141,4 @@ public void writeMemory( } finishBatch(); } - - /** - * Write to memory reading from a stream of data that includes context - * switches. Each section is a "header" of memory address (integer), - * chip x and y (each a short) and the number of words, followed by the - * words themselves. The number of words does not have to be short enough - * to fit in a packet; this will be managed by this function. - * - * @param core - * The location to send the messages to. - * @param data - * The stream of data to write. - * @throws IOException - * If anything goes wrong with networking or the input stream. - * @throws ProcessException - * If SpiNNaker rejects a message. - * @throws InterruptedException - * If the communications were interrupted. - */ - public void writeMemoryStream(HasCoreLocation core, InputStream data) - throws IOException, ProcessException, InterruptedException { - System.err.println("Reading information"); - DataInputStream input = new DataInputStream(data); - var baseAddress = new MemoryLocation(input.readInt()); - var targetChip = new ChipLocation(input.readShort(), input.readShort()); - var nWords = input.readInt(); - - System.err.println("Writing " + nWords + " words to " + baseAddress - + " on " + targetChip); - - var writePosition = baseAddress; - var nBytesRemaining = nWords * WORD_SIZE; - while (nBytesRemaining > 0) { - // One buffer per message; lifetime extends until batch end - System.err.println("Reading data; " + nBytesRemaining + " remaining"); - var tmp = read(input, allocate(UDP_MESSAGE_MAX_SIZE), - nBytesRemaining); - if (tmp == null) { - break; - } - System.err.println("Sending request"); - sendRequest(new SendMCDataRequest(core, targetChip, writePosition, - tmp)); - writePosition = writePosition.add(tmp.remaining()); - nBytesRemaining -= tmp.remaining(); - } - System.err.println("Finishing send"); - finishBatch(); - System.err.println("Finished!"); - } } diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 3e6ce67887..8b0fae95d5 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -24,14 +24,11 @@ import static java.util.stream.Collectors.toList; import static org.slf4j.LoggerFactory.getLogger; import static uk.ac.manchester.spinnaker.messages.Constants.NBBY; -import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; -import static uk.ac.manchester.spinnaker.messages.Constants.UDP_MESSAGE_MAX_SIZE; import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; import static uk.ac.manchester.spinnaker.utils.UnitConstants.MEGABYTE; import static uk.ac.manchester.spinnaker.utils.UnitConstants.NSEC_PER_SEC; import java.io.BufferedWriter; -import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -218,20 +215,19 @@ private void loadBoard(Ethernet board, DSEStorage storage) } log.info("loading data onto {} cores on board", cores.size()); var gather = gathererForChip.get(board.location); - try (var worker = new FastBoardWorker(txrx, board, storage, gather)) { + var worker = new FastBoardWorker(txrx, board, storage, gather); + for (var xyp : cores) { + worker.mallocCore(xyp); + } + try (var routers = worker.systemRouterTables(); + var context = worker.dontDropPackets(gather)) { for (var xyp : cores) { - worker.mallocCore(xyp); - } - try (var routers = worker.systemRouterTables(); - var context = worker.dontDropPackets(gather)) { - for (var xyp : cores) { - worker.loadCore(xyp); - } - log.info("finished sending data in for this board"); - } catch (Exception e) { - log.warn("failure in core loading", e); - throw e; + worker.loadCore(xyp); } + log.info("finished sending data in for this board"); + } catch (Exception e) { + log.warn("failure in core loading", e); + throw e; } } @@ -347,45 +343,15 @@ private static List describeChunk(Chunk chunk) { * @author Donal Fellows * @author Alan Stokes */ - private class FastBoardWorker extends BoardWorker implements AutoCloseable { - - private final CoreLocation ethernet; - - private final PipedInputStream input; - - private final DataOutputStream output; - - private final Thread outputThread; - + private class FastBoardWorker extends BoardWorker { + private HasCoreLocation ethernet; FastBoardWorker(TransceiverInterface txrx, Ethernet board, DSEStorage storage, Gather gather) throws IOException, ProcessException, InterruptedException, StorageException { super(txrx, board, storage); - System.err.println("Making fast board worker for " + board); - ethernet = new CoreLocation(board.location, gather.getP()); - input = new PipedInputStream(); - output = new DataOutputStream(new PipedOutputStream(input)); - outputThread = new Thread(() -> { - while (true) { - try { - System.err.println(this + ": Write multicast stream..."); - txrx.writeMemoryMulticastStream(ethernet, input); - System.err.println(this + ": Finish write multicast stream"); - } catch (Exception e) { - e.printStackTrace(); - } - } - }); - outputThread.start(); - } - - @Override - public void close() throws IOException, InterruptedException { - output.close(); - outputThread.join(); - input.close(); + this.ethernet = new CoreLocation(board.location, gather.getP()); } /** @@ -514,31 +480,31 @@ SystemRouterTableContext systemRouterTables() * If communications are interrupted. */ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, - ByteBuffer content) - throws IOException, InterruptedException, ProcessException { - - int boardLocalX = core.getX() - ethernet.getX(); - if (boardLocalX < 0) { - boardLocalX += machine.maxChipX() + 1; - } - int boardLocalY = core.getY() - ethernet.getY(); - if (boardLocalY < 0) { - boardLocalY += machine.maxChipY() + 1; + ByteBuffer data) + throws IOException, InterruptedException { + + try (var input = new PipedInputStream(data.remaining()); + var output = new PipedOutputStream(input)) { + byte[] transfer = new byte[data.remaining()]; + data.get(transfer); + output.write(transfer); + output.flush(); + output.close(); + int boardLocalX = core.getX() - ethernet.getX(); + if (boardLocalX < 0) { + boardLocalX += machine.maxChipX() + 1; + } + int boardLocalY = core.getY() - ethernet.getY(); + if (boardLocalY < 0) { + boardLocalY += machine.maxChipY() + 1; + } + var boardLocal = new CoreLocation(boardLocalX, boardLocalY, + core.getP()); + txrx.writeMemoryMulticast(ethernet, boardLocal, baseAddress, + input); + } catch (ProcessException e) { + throw new IOException(e); } - var data = content.duplicate(); - var nBytes = data.remaining(); - byte[] transfer = new byte[nBytes]; - data.get(transfer); - output.writeInt(baseAddress.address); - output.writeShort(boardLocalX); - output.writeShort(boardLocalY); - output.writeInt(nBytes / WORD_SIZE); - output.flush(); - System.err.println("Writing " + nBytes + " bytes to " + baseAddress); - output.write(transfer); - output.flush(); - System.err.println("Finished write"); - } } } From b1a192fe9e6488d2515abcbc73f2eeee50bd5ef5 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 7 Dec 2023 14:27:09 +0000 Subject: [PATCH 23/27] Revert "Move data through a stream" This reverts commit df1614bf7a1796c8139eff6c4cbaf12ba3216fa4. --- .../spinnaker/transceiver/Transceiver.java | 10 ------- .../transceiver/TransceiverInterface.java | 27 ------------------- .../dse/FastMCExecuteDataSpecification.java | 13 ++------- 3 files changed, 2 insertions(+), 48 deletions(-) diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java index e61ec6b4bd..edfed1d6eb 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/Transceiver.java @@ -2096,16 +2096,6 @@ public void writeMemoryMulticast(HasCoreLocation core, core, targetCore, baseAddress, data); } - @Override - @ParallelSafe - public void writeMemoryMulticast(HasCoreLocation core, - HasCoreLocation targetCore, MemoryLocation baseAddress, - InputStream data) - throws IOException, ProcessException, InterruptedException { - new WriteMemoryByMulticastProcess(scpSelector, this).writeMemory( - core, targetCore, baseAddress, data); - } - @Override @CheckReturnValue @ParallelSafe diff --git a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java index 95f5cb4f29..232587757a 100644 --- a/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java +++ b/SpiNNaker-comms/src/main/java/uk/ac/manchester/spinnaker/transceiver/TransceiverInterface.java @@ -1894,33 +1894,6 @@ void writeMemoryMulticast(@Valid HasCoreLocation core, @NotNull MemoryLocation baseAddress, @NotNull ByteBuffer data) throws IOException, ProcessException, InterruptedException; - /** - * Write to the board via multicast on the Ethernet chip. - * - * @param core - * The coordinates of the Ethernet core containing the advanced - * monitor support - * @param targetCore - * The coordinates of the core where the memory is that is to be - * written to - * @param baseAddress - * The address in SDRAM where the region of memory is to be - * written - * @param data - * The stream of data to be written. - * @throws IOException - * If anything goes wrong with networking. - * @throws ProcessException - * If SpiNNaker rejects a message. - * @throws InterruptedException - * If the communications were interrupted. - */ - @ParallelSafe - void writeMemoryMulticast(@Valid HasCoreLocation core, - @Valid HasCoreLocation targetCore, - @NotNull MemoryLocation baseAddress, @NotNull InputStream data) - throws IOException, ProcessException, InterruptedException; - /** * Write to the user0 register of a core. * diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index 8b0fae95d5..f64b6bb073 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -33,8 +33,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; import java.io.PrintWriter; import java.net.URISyntaxException; import java.nio.ByteBuffer; @@ -482,14 +480,7 @@ SystemRouterTableContext systemRouterTables() private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, ByteBuffer data) throws IOException, InterruptedException { - - try (var input = new PipedInputStream(data.remaining()); - var output = new PipedOutputStream(input)) { - byte[] transfer = new byte[data.remaining()]; - data.get(transfer); - output.write(transfer); - output.flush(); - output.close(); + try { int boardLocalX = core.getX() - ethernet.getX(); if (boardLocalX < 0) { boardLocalX += machine.maxChipX() + 1; @@ -501,7 +492,7 @@ private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, var boardLocal = new CoreLocation(boardLocalX, boardLocalY, core.getP()); txrx.writeMemoryMulticast(ethernet, boardLocal, baseAddress, - input); + data); } catch (ProcessException e) { throw new IOException(e); } From 72101ce8d0af0fed42c11752832c9593e7e52d3c Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Fri, 8 Dec 2023 16:22:09 +0000 Subject: [PATCH 24/27] Remove bits no longer needed --- .../front_end/CommandLineInterface.java | 65 -- .../front_end/dse/FastDataInCommandID.java | 66 -- .../front_end/dse/FastDataInProtocol.java | 202 ----- .../dse/FastExecuteDataSpecification.java | 793 ------------------ .../spinnaker/front_end/TestFrontEnd.java | 10 +- 5 files changed, 5 insertions(+), 1131 deletions(-) delete mode 100644 SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastDataInCommandID.java delete mode 100644 SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastDataInProtocol.java delete mode 100644 SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastExecuteDataSpecification.java diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/CommandLineInterface.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/CommandLineInterface.java index a92e7c1612..7ac0d01ae1 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/CommandLineInterface.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/CommandLineInterface.java @@ -23,7 +23,6 @@ import static uk.ac.manchester.spinnaker.alloc.client.SpallocClientFactory.getJobFromProxyInfo; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DOWNLOAD_DESC; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DSE_APP_DESC; -import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DSE_MON_DESC; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DSE_MON_DESC_MC; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.DSE_SYS_DESC; import static uk.ac.manchester.spinnaker.front_end.CommandDescriptions.GATHER_DESC; @@ -71,7 +70,6 @@ import uk.ac.manchester.spinnaker.front_end.download.RecordingRegionDataGatherer; import uk.ac.manchester.spinnaker.front_end.download.request.Gather; import uk.ac.manchester.spinnaker.front_end.download.request.Placement; -import uk.ac.manchester.spinnaker.front_end.dse.FastExecuteDataSpecification; import uk.ac.manchester.spinnaker.front_end.dse.FastMCExecuteDataSpecification; import uk.ac.manchester.spinnaker.front_end.dse.HostExecuteDataSpecification; import uk.ac.manchester.spinnaker.front_end.iobuf.IobufRequest; @@ -195,21 +193,6 @@ HostExecuteDataSpecification create(TransceiverInterface txrx, */ static HostDSEFactory hostFactory = HostExecuteDataSpecification::new; - @FunctionalInterface - interface FastDSEFactory { - FastExecuteDataSpecification create(TransceiverInterface txrx, - Machine machine, List gatherers, File reportDir, - DSEDatabaseEngine db) - throws IOException, SpinnmanException, StorageException, - ExecutionException, InterruptedException, URISyntaxException; - } - - /** - * Makes {@link FastExecuteDataSpecification} instances. Allows for - * injection of debugging tooling. - */ - static FastDSEFactory fastFactory = FastExecuteDataSpecification::new; - @FunctionalInterface interface FastMCDSEFactory { FastMCExecuteDataSpecification create(TransceiverInterface txrx, @@ -272,54 +255,6 @@ public void runDSEUploadingViaClassicTransfer(Machine machine, } } - /** - * Run the data specifications in parallel. - * - * @param gatherers - * List of descriptions of gatherers. - * @param machine - * Description of overall machine. - * @param dsFile - * Path to the dataspec database - * @param runFolder - * Directory containing per-run information. - * @param reportFolder - * Directory containing reports. If {@link Optional#empty()}, no - * report will be written. - * @throws IOException - * If the communications fail. - * @throws SpinnmanException - * If a BMP is uncontactable or SpiNNaker rejects a message. - * @throws StorageException - * If the database is in an illegal state. - * @throws ExecutionException - * If there was a problem in the parallel queue. - * @throws InterruptedException - * If the wait for everything to complete is interrupted. - * @throws URISyntaxException - * If a proxy URI is provided but invalid. - */ - @Command(name = "dse_app_mon", description = DSE_MON_DESC) - public void runDSEForAppCoresUploadingViaMonitorStreaming( - @Mixin GatherersParam gatherers, - @Mixin MachineParam machine, - @Mixin DsFileParam dsFile, - @Mixin RunFolderParam runFolder, - @Parameters(description = REPORT, arity = "0..1", index = "3") - Optional reportFolder) - throws IOException, SpinnmanException, StorageException, - ExecutionException, InterruptedException, URISyntaxException { - setLoggerDir(runFolder.get()); - var db = getDataSpecDB(dsFile.get()); - var job = getJob(db); - - try (var txrx = getTransceiver(machine.get(), job); - var dseExec = fastFactory.create(txrx, machine.get(), - gatherers.get(), reportFolder.orElse(null), db)) { - dseExec.loadCores(); - } - } - /** * Run the data specifications in parallel using monitors and multicast. * diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastDataInCommandID.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastDataInCommandID.java deleted file mode 100644 index 6232432c4c..0000000000 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastDataInCommandID.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.manchester.spinnaker.front_end.dse; - -import static uk.ac.manchester.spinnaker.utils.CollectionUtils.makeEnumBackingMap; - -import java.util.Map; - -/** - * Command IDs for the SDP packets for data in. - * - * @author Donal Fellows - */ -public enum FastDataInCommandID { - /** Host to Gatherer: start accepting data bound for location. */ - SEND_DATA_TO_LOCATION(200), - /** Host to Gatherer: more data with sequence number. */ - SEND_SEQ_DATA(2000), - /** Host to Gatherer: all data transmitted. */ - SEND_TELL_DATA_IN(2001), - /** Gatherer to host: there are missing sequence numbers. */ - RECEIVE_MISSING_SEQ_DATA_IN(2002), - /** Gatherer to host: all present and correct. */ - RECEIVE_FINISHED_DATA_IN(2003); - - private static final Map MAP = - makeEnumBackingMap(values(), v -> v.value); - - /** The protocol ID of this constant. */ - public final int value; - - FastDataInCommandID(int value) { - this.value = value; - } - - /** - * Get a constant by its protocol ID. - * - * @param value - * The protocol ID - * @return The matching constant. - * @throws IllegalArgumentException - * if the value isn't one of the ones accepted by this class. - */ - public static FastDataInCommandID forValue(int value) { - var id = MAP.get(value); - if (id == null) { - throw new IllegalArgumentException( - "unexpected command code: " + value); - } - return id; - } -} diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastDataInProtocol.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastDataInProtocol.java deleted file mode 100644 index 0b7b4fca4d..0000000000 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastDataInProtocol.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.manchester.spinnaker.front_end.dse; - -import static java.lang.Integer.toUnsignedLong; -import static java.lang.Math.min; -import static java.lang.String.format; -import static java.nio.ByteBuffer.allocate; -import static java.nio.ByteOrder.LITTLE_ENDIAN; -import static uk.ac.manchester.spinnaker.front_end.dse.FastDataInCommandID.SEND_DATA_TO_LOCATION; -import static uk.ac.manchester.spinnaker.front_end.dse.FastDataInCommandID.SEND_SEQ_DATA; -import static uk.ac.manchester.spinnaker.front_end.dse.FastDataInCommandID.SEND_TELL_DATA_IN; -import static uk.ac.manchester.spinnaker.messages.Constants.SDP_PAYLOAD_WORDS; -import static uk.ac.manchester.spinnaker.messages.Constants.WORD_SIZE; -import static uk.ac.manchester.spinnaker.messages.sdp.SDPHeader.Flag.REPLY_NOT_EXPECTED; -import static uk.ac.manchester.spinnaker.messages.sdp.SDPPort.GATHERER_DATA_SPEED_UP; -import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.slice; -import static uk.ac.manchester.spinnaker.utils.MathUtils.ceildiv; - -import java.nio.ByteBuffer; - -import uk.ac.manchester.spinnaker.machine.ChipLocation; -import uk.ac.manchester.spinnaker.machine.HasChipLocation; -import uk.ac.manchester.spinnaker.machine.HasCoreLocation; -import uk.ac.manchester.spinnaker.machine.Machine; -import uk.ac.manchester.spinnaker.machine.MemoryLocation; -import uk.ac.manchester.spinnaker.messages.sdp.SDPHeader; -import uk.ac.manchester.spinnaker.messages.sdp.SDPLocation; -import uk.ac.manchester.spinnaker.messages.sdp.SDPMessage; - -/** - * Manufactures Fast Data In protocol messages. - * - * @author Donal Fellows - * @author Alan Stokes - */ -class FastDataInProtocol { - /** Items of data a SDP packet can hold when SCP header removed. */ - static final int BYTES_PER_FULL_PACKET = SDP_PAYLOAD_WORDS * WORD_SIZE; - - // 272 bytes as removed SCP header - - /** - * size of the location data packet (command, transaction id, start sdram - * address, x and y, and max packet number. - */ - static final int BYTES_FOR_LOCATION_PACKET = 5 * WORD_SIZE; - - /** - * Offset where data in starts on first command (command, transaction id, - * seq_number), in bytes. - */ - static final int OFFSET_AFTER_COMMAND_AND_KEY = 3 * WORD_SIZE; - - /** Size for data to store when packet with command and key. */ - static final int DATA_IN_FULL_PACKET_WITH_KEY = - BYTES_PER_FULL_PACKET - OFFSET_AFTER_COMMAND_AND_KEY; - - /** - * size for data to store when sending tell packet (command id, transaction - * id). - */ - static final int BYTES_FOR_TELL_PACKET = 2 * WORD_SIZE; - - private final HasCoreLocation gathererCore; - - private final HasChipLocation boardLocalDestination; - - /** - * Create an instance of the protocol for talking to a particular extra - * monitor core on a particular board. - * - * @param machine - * The machine containing the board. - * @param gathererCore - * The gatherer core on the board that messages will be routed - * via. - * @param monitorChip - * The extra monitor core on the board that is the destination - * for the messages. - */ - FastDataInProtocol(Machine machine, HasCoreLocation gathererCore, - HasChipLocation monitorChip) { - this.gathererCore = gathererCore; - - int boardLocalX = monitorChip.getX() - gathererCore.getX(); - if (boardLocalX < 0) { - boardLocalX += machine.maxChipX() + 1; - } - int boardLocalY = monitorChip.getY() - gathererCore.getY(); - if (boardLocalY < 0) { - boardLocalY += machine.maxChipY() + 1; - } - this.boardLocalDestination = new ChipLocation(boardLocalX, boardLocalY); - } - - private SDPHeader header() { - return new SDPHeader(REPLY_NOT_EXPECTED, new SDPLocation(gathererCore), - GATHERER_DATA_SPEED_UP.value); - } - - /** - * @param baseAddress - * Where the data is to be written. - * @param numPackets - * How many SDP packets will be sent. - * @param transactionId - * The transaction id of this stream. - * @return The message indicating the start of the data. - */ - SDPMessage dataToLocation(MemoryLocation baseAddress, int numPackets, - int transactionId) { - var payload = allocate(BYTES_FOR_LOCATION_PACKET).order(LITTLE_ENDIAN); - payload.putInt(SEND_DATA_TO_LOCATION.value); - payload.putInt(transactionId); - payload.putInt(baseAddress.address); - payload.putShort((short) boardLocalDestination.getY()); - payload.putShort((short) boardLocalDestination.getX()); - payload.putInt(numPackets - 1); - payload.flip(); - return new SDPMessage(header(), payload); - } - - /** - * @param data - * The overall data to be transmitted. - * @param seqNum - * The sequence number of this chunk. - * - * @param transactionId - * The transaction id for this stream. - * @return The message containing a chunk of the data. - * @throws RuntimeException - * If the sequence number is nonsense. - */ - SDPMessage seqData(ByteBuffer data, int seqNum, int transactionId) { - var payload = allocate(BYTES_PER_FULL_PACKET).order(LITTLE_ENDIAN); - int position = calculatePositionFromSequenceNumber(seqNum); - if (position >= data.limit()) { - throw new RuntimeException(format( - "attempt to write off end of buffer due to " - + "over-large sequence number (%d) given " - + "that only %d bytes are to be sent", - seqNum, toUnsignedLong(data.limit()))); - } - payload.putInt(SEND_SEQ_DATA.value); - payload.putInt(transactionId); - payload.putInt(seqNum); - putBuffer(data, position, payload); - return new SDPMessage(header(), payload); - } - - private int putBuffer(ByteBuffer data, int position, ByteBuffer payload) { - var slice = slice(data, position, - min(data.remaining() - position, payload.remaining())); - payload.put(slice).flip(); - return slice.position(); - } - - private int calculatePositionFromSequenceNumber(int seqNum) { - return DATA_IN_FULL_PACKET_WITH_KEY * seqNum; - } - - /** - * generates the tell message. - * - * @param transactionId - * The transaction id for this stream. - * @return The message indicating the end of the data. - */ - SDPMessage tellDataIn(int transactionId) { - var payload = allocate(BYTES_FOR_TELL_PACKET).order(LITTLE_ENDIAN); - payload.putInt(SEND_TELL_DATA_IN.value); - payload.putInt(transactionId); - payload.flip(); - return new SDPMessage(header(), payload); - } - - /** - * Computes the number of packets required to send the given data. - * - * @param data - * The data being sent. (This operation only reads.) - * @return The number of packets (i.e. 1 more than the max sequence number). - */ - static int computeNumPackets(ByteBuffer data) { - return ceildiv(data.remaining(), DATA_IN_FULL_PACKET_WITH_KEY); - } -} diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastExecuteDataSpecification.java deleted file mode 100644 index cb167c3377..0000000000 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastExecuteDataSpecification.java +++ /dev/null @@ -1,793 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.manchester.spinnaker.front_end.dse; - -import static difflib.DiffUtils.diff; -import static java.lang.Integer.toUnsignedLong; -import static java.lang.String.format; -import static java.lang.System.getProperty; -import static java.lang.System.nanoTime; -import static java.nio.ByteOrder.LITTLE_ENDIAN; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.stream.Collectors.toList; -import static java.util.stream.IntStream.range; -import static org.slf4j.LoggerFactory.getLogger; -import static uk.ac.manchester.spinnaker.front_end.dse.FastDataInProtocol.computeNumPackets; -import static uk.ac.manchester.spinnaker.messages.Constants.NBBY; -import static uk.ac.manchester.spinnaker.utils.ByteBufferUtils.sliceUp; -import static uk.ac.manchester.spinnaker.utils.UnitConstants.MEGABYTE; -import static uk.ac.manchester.spinnaker.utils.UnitConstants.NSEC_PER_SEC; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.net.SocketTimeoutException; -import java.net.URISyntaxException; -import java.nio.ByteBuffer; -import java.nio.IntBuffer; -import java.util.ArrayDeque; -import java.util.BitSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; - -import com.google.errorprone.annotations.CheckReturnValue; -import com.google.errorprone.annotations.MustBeClosed; - -import difflib.ChangeDelta; -import difflib.Chunk; -import difflib.DeleteDelta; -import difflib.InsertDelta; -import uk.ac.manchester.spinnaker.front_end.NoDropPacketContext; -import uk.ac.manchester.spinnaker.front_end.download.request.Gather; -import uk.ac.manchester.spinnaker.front_end.download.request.Monitor; -import uk.ac.manchester.spinnaker.machine.ChipLocation; -import uk.ac.manchester.spinnaker.machine.CoreSubsets; -import uk.ac.manchester.spinnaker.machine.HasChipLocation; -import uk.ac.manchester.spinnaker.machine.HasCoreLocation; -import uk.ac.manchester.spinnaker.machine.Machine; -import uk.ac.manchester.spinnaker.machine.MemoryLocation; -import uk.ac.manchester.spinnaker.storage.DSEDatabaseEngine; -import uk.ac.manchester.spinnaker.storage.DSEStorage; -import uk.ac.manchester.spinnaker.storage.DSEStorage.Ethernet; -import uk.ac.manchester.spinnaker.storage.StorageException; -import uk.ac.manchester.spinnaker.transceiver.ProcessException; -import uk.ac.manchester.spinnaker.transceiver.TransceiverInterface; -import uk.ac.manchester.spinnaker.utils.MathUtils; - -/** - * Implementation of the Data Specification Executor that uses the Fast Data In - * protocol to upload the results to a SpiNNaker machine. - * - * @author Donal Fellows - * @author Alan Stokes - */ -public class FastExecuteDataSpecification extends ExecuteDataSpecification { - private static final Logger log = - getLogger(FastExecuteDataSpecification.class); - - private static final String SPINNAKER_COMPARE_UPLOAD = - getProperty("spinnaker.compare.upload"); - - private static final String IN_REPORT_NAME = - "speeds_gained_in_speed_up_process.tsv"; - - private static final int TIMEOUT_RETRY_LIMIT = 100; - - /** flag for saying missing all SEQ numbers. */ - private static final int FLAG_FOR_MISSING_ALL_SEQUENCES = 0xFFFFFFFE; - - /** Sequence number that marks the end of a sequence number stream. */ - private static final int MISSING_SEQS_END = -1; - - private final Map gathererForChip; - - private final Map monitorForChip; - - private final Map monitorsForBoard; - - private boolean writeReports = false; - - private File reportPath = null; - - /** - * Create an instance of this class. - * - * @param txrx - * The transceiver for talking to the SpiNNaker machine. - * @param machine - * The SpiNNaker machine description. - * @param gatherers - * The description of where the gatherers and monitors are. - * @param reportDir - * Where to write reports, or {@code null} if no reports are to - * be written. - * @param db - * The DSE Database. - * @throws IOException - * If IO goes wrong. - * @throws ProcessException - * If SpiNNaker rejects a message. - * @throws InterruptedException - * If communications are interrupted. - * @throws URISyntaxException - * If the proxy URI is provided but not valid. - * @throws StorageException - * If there is an error reading the database. - * @throws IllegalStateException - * If something really strange occurs with talking to the BMP; - * this constructor should not be doing that! - */ - @MustBeClosed - public FastExecuteDataSpecification(TransceiverInterface txrx, - Machine machine, List gatherers, File reportDir, - DSEDatabaseEngine db) throws IOException, ProcessException, - InterruptedException, StorageException, URISyntaxException { - super(txrx, machine, db); - if (SPINNAKER_COMPARE_UPLOAD != null) { - log.warn( - "detailed comparison of uploaded data enabled; " - + "this may destabilize the protocol"); - } - - if (reportDir != null) { - writeReports = true; - reportPath = new File(reportDir, IN_REPORT_NAME); - } - - gathererForChip = new HashMap<>(); - monitorForChip = new HashMap<>(); - monitorsForBoard = new HashMap<>(); - - buildMaps(gatherers); - } - - /** - * Construct the internal mappings for gatherers and monitors. - * - * @param gatherers - * The descriptions of whether the gatherers are located. - * @throws IOException - * If IDs can't be read from the machine for network reasons. - * @throws ProcessException - * If IDs can't be read from the machine for machine reasons. - * @throws InterruptedException - * If we are interrupted. - */ - protected void buildMaps(List gatherers) - throws IOException, ProcessException, InterruptedException { - for (var g : gatherers) { - g.updateTransactionIdFromMachine(txrx); - var gathererChip = g.asChipLocation(); - gathererForChip.put(gathererChip, g); - var boardMonitorCores = monitorsForBoard - .computeIfAbsent(gathererChip, __ -> new CoreSubsets()); - for (var m : g.getMonitors()) { - var monitorChip = m.asChipLocation(); - gathererForChip.put(monitorChip, g); - monitorForChip.put(monitorChip, m); - boardMonitorCores.addCore(m.asCoreLocation()); - } - } - } - - /** - * Execute all application data specifications that a particular connection - * knows about, storing back in the database the information collected about - * those executions. Data is transferred using the Fast Data In protocol. - *

- * Cannot load data for system cores; those are used by the implementation - * of this protocol. - * - * @throws StorageException - * If the database can't be talked to. - * @throws IOException - * If the transceiver can't talk to its sockets. - * @throws ProcessException - * If SpiNNaker rejects a message. - * @throws InterruptedException - * If communications are interrupted. - * @throws IllegalStateException - * If an unexpected exception occurs in any of the parallel - * tasks. - */ - public void loadCores() - throws StorageException, IOException, ProcessException, - InterruptedException { - var storage = db.getStorageInterface(); - processTasksInParallel(storage.listEthernetsToLoad(), board -> { - return () -> loadBoard(board, storage); - }); - } - - private void loadBoard(Ethernet board, DSEStorage storage) - throws IOException, ProcessException, StorageException, - InterruptedException { - var cores = storage.listCoresToLoad(board, false); - if (cores.isEmpty()) { - log.info("no cores need loading on board; skipping"); - return; - } - log.info("loading data onto {} cores on board", cores.size()); - var gather = gathererForChip.get(board.location); - try (var worker = new FastBoardWorker( - txrx, board, storage, gather)) { - for (var xyp : cores) { - worker.mallocCore(xyp); - } - try (var routers = worker.systemRouterTables(); - var context = worker.dontDropPackets(gather)) { - for (var xyp : cores) { - worker.loadCore(xyp); - } - log.info("finished sending data in for this board"); - } catch (Exception e) { - log.warn("failure in core loading", e); - throw e; - } - } - } - - /** - * Opens a file for writing text. - * - * @param file - * The file to open - * @param append - * Whether to open in append mode; if {@code false}, the file - * will be created or overwritten. - * @return The stream to use to do the writing. - * @throws IOException - * If anything goes wrong with creating or opening the file. - */ - private static PrintWriter open(File file, boolean append) - throws IOException { - return new PrintWriter(new BufferedWriter(new OutputStreamWriter( - new FileOutputStream(file, append), UTF_8))); - } - - /** - * Writes (part of) the report describing what data transfer rates were - * achieved. - * - * @param chip - * Which chip was the data bound for? - * @param timeDiff - * How long did the transfer take, in nanoseconds. - * @param size - * How many bytes were transferred? - * @param baseAddress - * Where were the bytes written to? - * @param missingNumbers - * What were the missing sequence numbers at each stage. - * @throws IOException - * If IO fails. - */ - public synchronized void writeReport(HasChipLocation chip, long timeDiff, - int size, MemoryLocation baseAddress, Object missingNumbers) - throws IOException { - if (!reportPath.exists()) { - try (var w = open(reportPath, false)) { - w.println("x" + "\ty" + "\tSDRAM address" + "\tsize/bytes" - + "\ttime taken/s" + "\ttransfer rate/(Mb/s)" - + "\tmissing sequence numbers"); - } - } - - float timeTaken = timeDiff / (float) NSEC_PER_SEC; - float megabits = (size * (long) NBBY) / (float) MEGABYTE; - String mbs; - if (timeDiff == 0) { - mbs = "unknown, below threshold"; - } else { - mbs = format("%f", megabits / timeTaken); - } - try (var w = open(reportPath, true)) { - w.printf("%d\t%d\t%s\t%d\t%f\t%s\t%s\n", chip.getX(), chip.getY(), - baseAddress, toUnsignedLong(size), timeTaken, mbs, - missingNumbers); - } - } - - private static void compareBuffers(ByteBuffer original, - ByteBuffer downloaded) { - for (int i = 0; i < original.remaining(); i++) { - if (original.get(i) != downloaded.get(i)) { - log.error("downloaded buffer contents different"); - for (var delta : diff(list(original), list(downloaded)) - .getDeltas()) { - if (delta instanceof ChangeDelta) { - var delete = delta.getOriginal(); - var insert = delta.getRevised(); - log.warn( - "swapped {} bytes (SCP) for {} (gather) " - + "at {}->{}", - delete.getLines().size(), - insert.getLines().size(), delete.getPosition(), - insert.getPosition()); - log.info("change {} -> {}", describeChunk(delete), - describeChunk(insert)); - } else if (delta instanceof DeleteDelta) { - var delete = delta.getOriginal(); - log.warn("gather deleted {} bytes at {}", - delete.getLines().size(), delete.getPosition()); - log.info("delete {}", describeChunk(delete)); - } else if (delta instanceof InsertDelta) { - var insert = delta.getRevised(); - log.warn("gather inserted {} bytes at {}", - insert.getLines().size(), insert.getPosition()); - log.info("insert {}", describeChunk(insert)); - } - } - break; - } - } - } - - private static List list(ByteBuffer buffer) { - return sliceUp(buffer, 1).map(ByteBuffer::get).toList(); - } - - private static List describeChunk(Chunk chunk) { - return chunk.getLines().stream().map(MathUtils::hexbyte) - .collect(toList()); - } - - /** - * The worker class that handles a particular board of a SpiNNaker machine. - * Instances of this class are only ever used from a single thread. - * - * @author Donal Fellows - * @author Alan Stokes - */ - private class FastBoardWorker extends BoardWorker implements AutoCloseable { - private ThrottledConnection connection; - - private MissingRecorder missingSequenceNumbers; - - private BoardLocal logContext; - - private Gather gather; - - @MustBeClosed - @SuppressWarnings("MustBeClosed") - FastBoardWorker(TransceiverInterface txrx, Ethernet board, - DSEStorage storage, Gather gather) - throws IOException, ProcessException, InterruptedException, - StorageException { - super(txrx, board, storage); - this.logContext = new BoardLocal(board.location); - this.connection = new ThrottledConnection(txrx, board, - gather.getIptag()); - this.gather = gather; - } - - @Override - public void close() throws IOException, ProcessException, - InterruptedException { - logContext.close(); - connection.close(); - } - - /** - * A list of bitfields. Knows how to install and uninstall itself from - * the general execution flow. - * - * @author Donal Fellows - */ - @SuppressWarnings("serial") - private class MissingRecorder extends ArrayDeque - implements AutoCloseable { - MissingRecorder() { - missingSequenceNumbers = this; - } - - @Override - public void close() { - missingSequenceNumbers = null; - } - - /** - * Give me a new bitfield, recorded in this class. - * - * @param expectedMax - * How big should the bitfield be? - * @return The bitfield. - */ - BitSet issueNew(int expectedMax) { - var s = new BitSet(expectedMax); - addLast(s); - return s; - } - - /** - * Issue the report based on what we recorded, if appropriate. - * - * @param core - * What core were we recording for? - * @param time - * How long did the loading take? - * @param size - * How much data was moved? - * @param addr - * Where on the core was the data moved to? - * @throws IOException - * If anything goes wrong with writing. - */ - void report(HasCoreLocation core, long time, int size, - MemoryLocation addr) throws IOException { - if (writeReports) { - writeReport(core, time, size, addr, this); - } - } - } - - /** - * Writes the contents of a region. Caller is responsible for ensuring - * this method has work to do. - * - * @param core - * Which core to write to. Does not need to refer to a - * monitor core. - * @param region - * The region to write. - * @param baseAddress - * Where to write the region. - * @param gather - * The information about where messages are routed via. - * @return How many bytes were actually written. - * @throws IOException - * If anything goes wrong with I/O. - * @throws ProcessException - * If SCAMP rejects the request. - * @throws InterruptedException - * If communications are interrupted. - */ - @Override - protected int writeRegion(HasCoreLocation core, ByteBuffer content, - MemoryLocation baseAddress) - throws IOException, ProcessException, InterruptedException { - int written = content.remaining(); - try (var recorder = new MissingRecorder()) { - long start = nanoTime(); - fastWrite(core, baseAddress, content); - long end = nanoTime(); - recorder.report( - core, end - start, content.limit(), baseAddress); - } - if (SPINNAKER_COMPARE_UPLOAD != null) { - var readBack = txrx.readMemory( - core, baseAddress, content.remaining()); - compareBuffers(content, readBack); - } - return written; - } - - /** - * Put the board in don't-drop-packets mode. - * - * @param core - * The core location of the gatherer for the board to set to - * don't drop packets. - * @return An object that, when closed, will put the board back in - * standard mode. - * @throws IOException - * If anything goes wrong with communication. - * @throws ProcessException - * If SpiNNaker rejects a message. - * @throws InterruptedException - * If communications are interrupted. - */ - @MustBeClosed - NoDropPacketContext dontDropPackets(Gather core) - throws IOException, ProcessException, InterruptedException { - return new NoDropPacketContext(txrx, - monitorsForBoard.get(board.location), core); - } - - /** - * Install the system router tables across the board. - * - * @return An object that, when closed, will put the board back in - * application mode. - * @throws IOException - * If anything goes wrong with communication. - * @throws ProcessException - * If SpiNNaker rejects a message. - * @throws InterruptedException - * If communications are interrupted. - */ - @MustBeClosed - SystemRouterTableContext systemRouterTables() - throws IOException, ProcessException, InterruptedException { - return new SystemRouterTableContext(txrx, - monitorsForBoard.get(board.location)); - } - - /** - * This is the implementation of the actual fast data in protocol. - * - * @param core - * Where the data is going to. - * @param baseAddress - * Whether the data will be written. - * @param data - * The data to be written. - * @throws IOException - * If IO fails. - * @throws InterruptedException - * If communications are interrupted. - */ - private void fastWrite(HasCoreLocation core, MemoryLocation baseAddress, - ByteBuffer data) - throws IOException, InterruptedException { - int timeoutCount = 0; - int numPackets = computeNumPackets(data); - var protocol = new GathererProtocol(core); - int transactionId = gather.getNextTransactionId(); - - outerLoop: while (true) { - // Do the initial blast of data - sendInitialPackets(baseAddress, data, protocol, transactionId, - numPackets); - /* - * Don't create a missing buffer until at least one packet has - * come back. - */ - BitSet missing = null; - - // Wait for confirmation and do required retransmits - innerLoop: while (true) { - try { - var buf = connection.receive(); - var received = buf.order(LITTLE_ENDIAN).asIntBuffer(); - timeoutCount = 0; // Reset the timeout counter - int command = received.get(); - try { - // read transaction id - var commandCode = - FastDataInCommandID.forValue(command); - int thisTransactionId = received.get(); - - // if wrong transaction id, ignore packet - if (thisTransactionId != transactionId) { - continue innerLoop; - } - - // Decide what to do with the packet - switch (commandCode) { - case RECEIVE_FINISHED_DATA_IN: - // We're done! - break outerLoop; - - case RECEIVE_MISSING_SEQ_DATA_IN: - if (!received.hasRemaining()) { - throw new BadDataInMessageException( - received.get(0), received); - } - log.debug( - "another packet (#{}) of missing " - + "sequence numbers;", - received.get(1)); - break; - default: - throw new BadDataInMessageException( - received.get(0), received); - } - - /* - * The currently received packet has missing - * sequence numbers. Accumulate and dispatch - * transactionId when we've got them all. - */ - if (missing == null) { - missing = missingSequenceNumbers.issueNew( - numPackets); - } - var flags = addMissedSeqNums( - received, missing, numPackets); - - /* - * Check that you've seen something that implies - * ready to retransmit. - */ - if (flags.seenAll || flags.seenEnd) { - retransmitMissingPackets(protocol, data, - missing, transactionId); - missing.clear(); - } - } catch (IllegalArgumentException e) { - log.error("Unexpected command code " + command - + " received from " - + connection.getLocation()); - } - } catch (SocketTimeoutException e) { - if (timeoutCount++ > TIMEOUT_RETRY_LIMIT) { - log.error( - "ran out of attempts on transaction {}" - + " due to timeouts.", - transactionId); - throw e; - } - /* - * If we never received a packet, we will never have - * created the buffer, so send everything again - */ - if (missing == null) { - log.debug("full timeout; resending initial " - + "packets for stream with transaction " - + "id {}", transactionId); - continue outerLoop; - } - log.info( - "timeout {} on transaction {} sending to {}" - + " via {}", - timeoutCount, transactionId, core, - gather.asCoreLocation()); - retransmitMissingPackets(protocol, data, missing, - transactionId); - missing.clear(); - } - } - } - } - - @CheckReturnValue - private SeenFlags addMissedSeqNums(IntBuffer received, BitSet seqNums, - int expectedMax) { - var flags = new SeenFlags(); - var addedEnd = ""; - var addedAll = ""; - int actuallyAdded = 0; - while (received.hasRemaining()) { - int num = received.get(); - - if (num == MISSING_SEQS_END) { - addedEnd = "and saw END marker"; - flags.seenEnd = true; - break; - } - if (num == FLAG_FOR_MISSING_ALL_SEQUENCES) { - addedAll = "by finding ALL missing marker"; - flags.seenAll = true; - for (int seqNum = 0; seqNum < expectedMax; seqNum++) { - seqNums.set(seqNum); - actuallyAdded++; - } - break; - } - - seqNums.set(num); - actuallyAdded++; - if (num < 0 || num > expectedMax) { - throw new CrazySequenceNumberException(num, received); - } - } - log.debug("added {} missed packets, {}{}", actuallyAdded, addedEnd, - addedAll); - return flags; - } - - private int sendInitialPackets(MemoryLocation baseAddress, - ByteBuffer data, GathererProtocol protocol, int transactionId, - int numPackets) throws IOException { - log.info("streaming {} bytes in {} packets using transaction {}", - data.remaining(), numPackets, transactionId); - log.debug("sending packet #{}", 0); - connection.send(protocol.dataToLocation(baseAddress, numPackets, - transactionId)); - for (int seqNum = 0; seqNum < numPackets; seqNum++) { - log.debug("sending packet #{}", seqNum); - connection.send(protocol.seqData(data, seqNum, transactionId)); - } - log.debug("sending terminating packet"); - connection.send(protocol.tellDataIn(transactionId)); - return numPackets; - } - - private void retransmitMissingPackets(GathererProtocol protocol, - ByteBuffer dataToSend, BitSet missingSeqNums, int transactionId) - throws IOException { - log.info("retransmitting {} packets", missingSeqNums.cardinality()); - - missingSeqNums.stream().forEach(seqNum -> { - log.debug("resending packet #{}", seqNum); - try { - connection.send(protocol.seqData(dataToSend, seqNum, - transactionId)); - } catch (IOException e) { - log.error( - "missing sequence packet with id {}-{} " - + "failed to transmit", - seqNum, transactionId, e); - } - }); - log.debug("sending terminating packet"); - connection.send(protocol.tellDataIn(transactionId)); - } - } - - /** - * Manufactures Fast Data In protocol messages. - * - * @author Donal Fellows - */ - private class GathererProtocol extends FastDataInProtocol { - private GathererProtocol(ChipLocation chip, boolean ignored) { - super(machine, gathererForChip.get(chip), monitorForChip.get(chip)); - } - - /** - * Create an instance of this for pushing data to a given chip's SDRAM. - * - * @param chip - * The chip where the data is to be pushed. What extra - * monitor and data gatherer to route it via are determined - * from the board context. - */ - GathererProtocol(HasChipLocation chip) { - this(chip.asChipLocation(), false); - } - } - - /** - * Contains flags for seen missing sequence numbers. - * - * @author Alan Stokes - */ - private static final class SeenFlags { - boolean seenEnd; - - boolean seenAll; - } - - /** - * Exception thrown when something mad comes back off SpiNNaker. - * - * @author Donal Fellows - */ - static class BadDataInMessageException extends RuntimeException { - private static final long serialVersionUID = 1L; - - BadDataInMessageException(int code, IntBuffer message) { - super("unexpected response code: " + toUnsignedLong(code)); - log.warn("bad message payload: {}", range(0, message.limit()) - .map(i -> message.get(i)).boxed().collect(toList())); - } - } - - /** - * Exception thrown when something mad comes back off SpiNNaker. - * - * @author Donal Fellows - * @author Alan Stokes - */ - static class CrazySequenceNumberException extends RuntimeException { - private static final long serialVersionUID = 1L; - - CrazySequenceNumberException(int remaining, IntBuffer message) { - super("crazy number of missing packets: " - + toUnsignedLong(remaining)); - log.warn("bad message payload: {}", range(0, message.limit()) - .map(i -> message.get(i)).boxed().collect(toList())); - } - } -} diff --git a/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java b/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java index 9e534359a6..26943768c1 100644 --- a/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java +++ b/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java @@ -31,7 +31,7 @@ import org.opentest4j.AssertionFailedError; import uk.ac.manchester.spinnaker.front_end.download.request.Gather; -import uk.ac.manchester.spinnaker.front_end.dse.FastExecuteDataSpecification; +import uk.ac.manchester.spinnaker.front_end.dse.FastMCExecuteDataSpecification; import uk.ac.manchester.spinnaker.front_end.dse.HostExecuteDataSpecification; import uk.ac.manchester.spinnaker.utils.ValueHolder; @@ -175,11 +175,11 @@ void testAdvancedDSE() throws Exception { var runFolder = "target/test/AdvancedDSE"; new File(runFolder).mkdirs(); - var saved = CommandLineInterface.fastFactory; + var saved = CommandLineInterface.fastMCFactory; var called = new ValueHolder<>("none"); try { - CommandLineInterface.fastFactory = (t, m, g, r, - db) -> new FastExecuteDataSpecification(t, m, g, r, null) { + CommandLineInterface.fastMCFactory = (t, m, g, r, + db) -> new FastMCExecuteDataSpecification(t, m, g, r, null) { @Override public void loadCores() { called.setValue("mon"); @@ -202,7 +202,7 @@ protected void buildMaps(List gatherers) { runFolder); assertEquals("mon", called.getValue()); } finally { - CommandLineInterface.fastFactory = saved; + CommandLineInterface.fastMCFactory = saved; } } From 316ac97cc05590195ea5af95e522e05305f603e8 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Mon, 11 Dec 2023 09:36:44 +0000 Subject: [PATCH 25/27] Fix tests --- .../uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java b/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java index 26943768c1..7f3c8722ed 100644 --- a/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java +++ b/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java @@ -98,7 +98,7 @@ void testHelp() throws Exception { var msg = tapSystemOutNormalized(() -> { runMainExpecting(0, "help"); }); - var requiredSubcommands = List.of("dse_app_mon", "gather"); + var requiredSubcommands = List.of("dse_app_mon_mc", "gather"); var requiredArgs = List.of("", ""); for (var cmd: requiredSubcommands) { assertContainsInOrder(msg, cmd); @@ -192,7 +192,7 @@ protected void buildMaps(List gatherers) { }; var msg = tapSystemErrNormalized(() -> { - runMainExpecting(2, "dse_app_mon"); + runMainExpecting(2, "dse_app_mon_mc"); }); assertContainsInOrder(msg, "", "", "", "[]"); From 0796ec98af43cb7e0f911ec7a91a3bc8bb10466a Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Mon, 11 Dec 2023 09:43:30 +0000 Subject: [PATCH 26/27] Missed one! --- .../uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java b/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java index 7f3c8722ed..ab07c39825 100644 --- a/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java +++ b/SpiNNaker-front-end/src/test/java/uk/ac/manchester/spinnaker/front_end/TestFrontEnd.java @@ -198,8 +198,8 @@ protected void buildMaps(List gatherers) { "", "[]"); assertEquals("none", called.getValue()); - runMainExpecting(0, "dse_app_mon", gatherFile, machineFile, dsFile, - runFolder); + runMainExpecting(0, "dse_app_mon_mc", gatherFile, machineFile, + dsFile, runFolder); assertEquals("mon", called.getValue()); } finally { CommandLineInterface.fastMCFactory = saved; From a489539cc7d11294706307fb903a03a6c1085118 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Mon, 22 Jul 2024 16:20:54 +0100 Subject: [PATCH 27/27] The constant has changed name... --- .../spinnaker/front_end/dse/FastMCExecuteDataSpecification.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java index f64b6bb073..7ae67d565d 100644 --- a/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java +++ b/SpiNNaker-front-end/src/main/java/uk/ac/manchester/spinnaker/front_end/dse/FastMCExecuteDataSpecification.java @@ -81,7 +81,7 @@ public class FastMCExecuteDataSpecification extends ExecuteDataSpecification { getProperty("spinnaker.compare.upload"); private static final String IN_REPORT_NAME = - "speeds_gained_in_speed_up_process.tsv"; + "speeds_gained_in_speed_up_process.rpt"; private final Map gathererForChip;