Split and combine data
Scatter, Gather, and Reduce
In real parallel programs, it's common to split data across processes, perform computations independently, then combine the results. MPI provides collective operations to do exactly this:
-
scatter: distributes chunks of data to each process -
gather: collects data from each process to one -
reduce: aggregates data using operations like sum, max, min, etc.
Scatter and Gather Example
We will send a list of numbers from process 0, give each process a slice, let them double it, and collect the results back.
- Python
- C language
Python file: mpi_scatter_gather.py
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# Only rank 0 creates the full data to scatter
if rank == 0:
data = [i for i in range(size)]
print(f"[Rank 0] Original data: {data}")
else:
data = None
# Create a variable to receive one element per process
recv_data = comm.scatter(data, root=0)
print(f"[Rank {rank}] Received value: {recv_data}")
# Each process doubles its value
processed = recv_data * 2
# Gather results at rank 0
gathered_data = comm.gather(processed, root=0)
if rank == 0:
print(f"[Rank 0] Gathered result: {gathered_data}")
SLURM script: mpi_scatter_gather.sh
#!/bin/bash
#SBATCH --job-name=mpi_for_beginner_scatter_gather
#SBATCH --output=mpi_for_beginner_scatter_gather.txt
#SBATCH --ntasks=4
#SBATCH --nodes=1
#SBATCH --partition=std
module load python/3.12.5
srun python mpi_scatter_gather.py
C file: mpi_scatter_gather.c
#include <stdio.h>
#include <mpi.h>
int main(int argc, char *argv[]) {
int rank, size;
int sendbuf[100]; // buffer for the rank 0
int recvbuf;
int result;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
if (rank == 0) {
for (int i = 0; i < size; i++) {
sendbuf[i] = i;
}
printf("[Rank 0] Original data: ");
for (int i = 0; i < size; i++) {
printf("%d ", sendbuf[i]);
}
printf("\n");
}
// Distribute one element per process
MPI_Scatter(sendbuf, 1, MPI_INT, &recvbuf, 1, MPI_INT, 0, MPI_COMM_WORLD);
printf("[Rank %d] Received value: %d\n", rank, recvbuf);
// Each process doubles its value
result = recvbuf * 2;
int gathered[100];
// Gather results
MPI_Gather(&result, 1, MPI_INT, gathered, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (rank == 0) {
printf("[Rank 0] Gathered result: ");
for (int i = 0; i < size; i++) {
printf("%d ", gathered[i]);
}
printf("\n");
}
MPI_Finalize();
return 0;
}
We compile the program with mpicc :
module load mpi/openmpi/4.1.8/gcc
mpicc -o mpi_scatter_gather mpi_scatter_gather.c
SLURM script: mpi_point_to_point.sh
#!/bin/bash
#SBATCH --job-name=mpi_for_beginner_scatter_gather
#SBATCH --output=mpi_for_beginner_scatter_gather.txt
#SBATCH --ntasks=4
#SBATCH --nodes=1
#SBATCH --partition=std
module load mpi/openmpi/4.1.8/gcc
srun ./mpi_scatter_gather
Output:

The order of the output lines may vary each time you run the program. This is because all MPI processes run in parallel and print at different times, independently of their rank.
Each process gets one element, processes it, and sends it back.
reduce: Aggregating Values
We compute the sum of values distributed across processes.
- Python
- C language
Python file: mpi_reduce.py
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
local_value = rank + 1
print(f"[Rank {rank}] Local value: {local_value}")
total = comm.reduce(local_value, op=MPI.SUM, root=0)
if rank == 0:
print(f"[Rank 0] Total sum: {total}")
SLURM script: mpi_reduce.sh
#!/bin/bash
#SBATCH --job-name=mpi_for_beginner_reduce
#SBATCH --output=mpi_for_beginner_reduce.txt
#SBATCH --ntasks=4
#SBATCH --nodes=1
#SBATCH --partition=std
module load python/3.12.5
srun python mpi_reduce.py
C file: mpi_reduce.c
#include <stdio.h>
#include <mpi.h>
int main(int argc, char *argv[]) {
int rank, size;
int local_value, total_sum;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
local_value = rank + 1;
printf("[Rank %d] Local value: %d\n", rank, local_value);
// Does the reduce (sum) of all local values to rank 0
MPI_Reduce(&local_value, &total_sum, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0) {
printf("[Rank 0] Total sum: %d\n", total_sum);
}
MPI_Finalize();
return 0;
}
We compile the program with mpicc :
module load mpi/openmpi/4.1.8/gcc
mpicc -o mpi_reduce mpi_reduce.c
SLURM script: mpi_point_to_point.sh
#!/bin/bash
#SBATCH --job-name=mpi_for_beginner_reduce
#SBATCH --output=mpi_for_beginner_reduce.txt
#SBATCH --ntasks=4
#SBATCH --nodes=1
#SBATCH --partition=std
module load mpi/openmpi/4.1.8/gcc
srun ./mpi_reduce
Output:
Summary
-
Use
scatterto divide data among processes -
Use
gatherto collect data back to one process -
Use
reduceto aggregate values (sum, min, max, etc.)