I would create a custom table that stored a) the user doing the rating b) the user being rated c) the rating d) the time of the rating. Between these 4, you should be able to store, aggregate, and effectively query your data with very little overhead or PHP processing.

update

note: this is pseudocode

function get_user_rating( $user_id ) {
    $rating = get_from_database(
        'SELECT AVG(rating)
        FROM ratings_table
        WHERE user_being_rated = '.$user_id
    );

    return $rating;
}

function add_user_rating( $rater, $rated, $rating ) {
    //check to see if there is already a rating to avoid duplicate records
    if( already_rated( $rater, $rated ) ) {
        //remove previous rating
        remove_rating( $rater, $rated );
    }
    //insert current rating
    insert_into_database(
        'INSERT INTO ratings_table
        ( user_doing_rating, user_being_rated, rating, time ) VALUE
        ( '.$rater.', '.$rated.', '.$rating.', CURTIME() )'
    );
}