@inproceedings{bbb51e85f8354a7a813ab7f445261b06,
title = "OSP: Overlapping computation and communication in parameter server for fast machine learning",
abstract = "When running in Parameter Server (PS), the Distributed Stochastic Gradient Descent (SGD) incurs significant communication delays because after pushing their updates, computing nodes (workers) have to wait for the global model to be communicated back from the master in every iteration. In this paper, we devise a new synchronization parallel mechanism named overlap synchronization parallel (OSP), in which the waiting time is removed by conducting computation and communication in an overlapped manner. We theoretically prove that our mechanism could achieve the same convergence rate compared to the sequential SGD for non-convex problems. Evaluations show that our mechanism significantly improves performance over the state-of-the-art ones, e.g., by 4× for both AlexNet and ResNet18 in terms of convergence speed.",
keywords = "Distributed, Machine learning, Parameter Server, SGD, Synchronization",
author = "Haozhao Wang and Song Guo and Ruixuan Li",
year = "2019",
month = aug,
day = "5",
doi = "10.1145/3337821.3337828",
language = "English",
series = "ACM International Conference Proceeding Series",
publisher = "Association for Computing Machinery",
pages = "1--10",
booktitle = "Proceedings of the 48th International Conference on Parallel Processing, ICPP 2019",
note = "48th International Conference on Parallel Processing, ICPP 2019 ; Conference date: 05-08-2019 Through 08-08-2019",
}